@dbx-tools/appkit-web-search 0.9.131 → 0.9.136
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -46
- package/index.ts +2 -1
- package/lib/index.d.ts +2 -1
- package/lib/index.js +2 -2
- package/lib/src/fetch.d.ts +31 -0
- package/lib/src/fetch.js +93 -19
- package/lib/src/plugin.d.ts +5 -0
- package/lib/src/plugin.js +6 -2
- package/lib/src/runtime.d.ts +5 -1
- package/lib/src/runtime.js +17 -5
- package/lib/src/search.js +79 -32
- package/package.json +7 -7
- package/src/fetch.ts +151 -22
- package/src/plugin.ts +18 -1
- package/src/runtime.ts +20 -3
- package/src/search.ts +97 -33
package/README.md
CHANGED
|
@@ -1,15 +1,13 @@
|
|
|
1
1
|
# @dbx-tools/appkit-web-search
|
|
2
2
|
|
|
3
|
-
Server-side web-search runtime, Mastra tools, and AppKit plugin.
|
|
4
|
-
|
|
5
3
|
Add `web_search` and `web_fetch` to an AppKit or Mastra backend. `web_search`
|
|
6
4
|
runs on the
|
|
7
5
|
[Databricks Model Serving native web-search tool](https://docs.databricks.com/aws/en/machine-learning/model-serving/web-search):
|
|
8
6
|
the model searches the web server-side and returns a synthesized answer plus the
|
|
9
|
-
sources it used
|
|
10
|
-
through [
|
|
7
|
+
sources it used without a third-party search API key. `web_fetch` reads a single page
|
|
8
|
+
through [got-scraping](https://www.npmjs.com/package/got-scraping).
|
|
11
9
|
|
|
12
|
-
## Quick
|
|
10
|
+
## Quick start
|
|
13
11
|
|
|
14
12
|
Register the plugin and give its tools to an agent:
|
|
15
13
|
|
|
@@ -30,7 +28,7 @@ await createApp({
|
|
|
30
28
|
});
|
|
31
29
|
```
|
|
32
30
|
|
|
33
|
-
|
|
31
|
+
Registration provides:
|
|
34
32
|
|
|
35
33
|
- AppKit plugin registration that resolves and logs the effective policy at boot.
|
|
36
34
|
- Two Mastra tools: `web_search` (answer + citations) and `web_fetch` (page
|
|
@@ -39,13 +37,13 @@ await createApp({
|
|
|
39
37
|
- `web_search` resolves a dedicated web-search-capable model - defaulting to Gemini,
|
|
40
38
|
then GPT - independently of the calling agent's chat model (which may not
|
|
41
39
|
support web search). Loose names (`"gemini"`, `"gpt"`) fuzzy-match the live
|
|
42
|
-
catalogue
|
|
40
|
+
catalogue through [@dbx-tools/model](../model).
|
|
43
41
|
- A built-in provider -> request-fragment map (OpenAI Responses API
|
|
44
42
|
`{"tools":[{"type":"web_search"}]}`, Gemini Chat Completions
|
|
45
43
|
`{"google_search":{}}`),
|
|
46
|
-
overridable per provider
|
|
47
|
-
- A named URL policy (`allowlist` or `unrestricted`) over a glob
|
|
48
|
-
built on [
|
|
44
|
+
overridable per provider with the `WEB_SEARCH_TOOLS` setting.
|
|
45
|
+
- A named URL policy (`allowlist` or `unrestricted`) over a glob allowlist
|
|
46
|
+
built on [@dbx-tools/path](../path)'s `match` matcher: `web_search` silently
|
|
49
47
|
filters disallowed citations, `web_fetch` refuses a disallowed URL, and the
|
|
50
48
|
mode in force is logged at boot.
|
|
51
49
|
- Per-tool approval gating: gate every call, or (for `web_fetch`) only calls
|
|
@@ -57,18 +55,24 @@ await createApp({
|
|
|
57
55
|
- Page and fallback-result HTML use parser-backed text conversion and selectors,
|
|
58
56
|
including complete HTML entity decoding and malformed-markup recovery.
|
|
59
57
|
|
|
60
|
-
##
|
|
58
|
+
## Choose a web provider
|
|
61
59
|
|
|
62
60
|
AppKit has no first-party web-search or page-fetch surface. Use this package when
|
|
63
61
|
an agent needs to look things up on the open web or read a URL the user pasted,
|
|
64
|
-
with a policy layer (
|
|
65
|
-
controls which sites are reachable and which calls pause for a human.
|
|
66
|
-
|
|
62
|
+
with a policy layer (allowlist + optional approval) around it so a deployment
|
|
63
|
+
controls which sites are reachable and which calls pause for a human. The
|
|
64
|
+
web-search tool resolves its own web-search-capable model, so an agent
|
|
67
65
|
running on any chat model (including one without web search) can still search. It
|
|
68
|
-
|
|
69
|
-
|
|
66
|
+
provides a Mastra tool pair and an AppKit plugin, like
|
|
67
|
+
[@dbx-tools/email](../email).
|
|
68
|
+
|
|
69
|
+
Use native Databricks web search when a supported GPT or Gemini endpoint is
|
|
70
|
+
deployed. Use the DuckDuckGo scrape fallback only when no capable endpoint is
|
|
71
|
+
available and unsynthesized search results are acceptable. Use `web_fetch` for a
|
|
72
|
+
specific public HTTPS page. For indexed application data, use
|
|
73
|
+
[@dbx-tools/search](../search) instead.
|
|
70
74
|
|
|
71
|
-
## Configure
|
|
75
|
+
## Configure the AppKit plugin
|
|
72
76
|
|
|
73
77
|
```ts
|
|
74
78
|
import { createApp, lakebase, server } from "@databricks/appkit";
|
|
@@ -101,7 +105,7 @@ the native plugin toolkit as shown above, which preserves that plugin's policy,
|
|
|
101
105
|
OBO scope, and AppKit execution chain. Approval, when enabled, requires Mastra
|
|
102
106
|
storage, so register `lakebase()` or configure storage in the Mastra plugin.
|
|
103
107
|
|
|
104
|
-
The plugin is also an AppKit `ToolProvider`, so a
|
|
108
|
+
The plugin is also an AppKit `ToolProvider`, so a native code agent can
|
|
105
109
|
take the same two tools without Mastra. Put the definition at
|
|
106
110
|
`server/agents/researcher/agent.ts` so AppKit discovers it:
|
|
107
111
|
|
|
@@ -132,16 +136,16 @@ Both tools are annotated `effect: "read"` and require user context, and neither
|
|
|
132
136
|
is auto-inheritable: an agent has to ask for them, because `web_fetch` reaches
|
|
133
137
|
whatever URL the model produces.
|
|
134
138
|
|
|
135
|
-
Native volume skills
|
|
139
|
+
Native volume skills run as the service principal. The 0.81
|
|
136
140
|
`skillCredentialMode: "obo"` option is not wired. Keep request-scoped writable
|
|
137
141
|
skills, remote sources, asynchronous Graphiti discovery, Sandbox/Monty,
|
|
138
142
|
Mastra memory, and durable approvals on `@dbx-tools/appkit-mastra`.
|
|
139
143
|
|
|
140
|
-
## Choose
|
|
144
|
+
## Choose the web-search model
|
|
141
145
|
|
|
142
146
|
The native web-search tool only runs on certain models, and it is provider-
|
|
143
147
|
specific. This package resolves deployed endpoints through
|
|
144
|
-
[
|
|
148
|
+
[@dbx-tools/model](../model) and applies this order:
|
|
145
149
|
|
|
146
150
|
1. Highest-ranked web-search-capable **Gemini**.
|
|
147
151
|
2. Without Gemini, a capable **GPT** matching the calling/requested model.
|
|
@@ -159,26 +163,26 @@ Serving binding, and it is the field the plugin declares in its manifest, so
|
|
|
159
163
|
The request's `model`, plugin `model`, and environment model values are used to
|
|
160
164
|
match the calling GPT after Gemini availability is checked.
|
|
161
165
|
When the tool runs through `@dbx-tools/appkit-mastra`, the host injects the
|
|
162
|
-
|
|
166
|
+
resolved chat model, overriding any different model argument generated
|
|
163
167
|
by the LLM.
|
|
164
168
|
|
|
165
|
-
Model resolution runs against the
|
|
166
|
-
[
|
|
169
|
+
Model resolution runs against the live workspace catalogue through
|
|
170
|
+
[@dbx-tools/model](../model), so it only picks an endpoint that is
|
|
167
171
|
actually deployed - a configured id that doesn't exist is never called.
|
|
168
172
|
|
|
169
|
-
### Scrape
|
|
173
|
+
### Scrape fallback
|
|
170
174
|
|
|
171
|
-
The native tool is
|
|
175
|
+
The native tool is preferred. When the workspace has no deployed
|
|
172
176
|
GPT/Gemini endpoint (so native web search can't run at all), the tool falls
|
|
173
|
-
back to a DuckDuckGo scrape
|
|
177
|
+
back to a DuckDuckGo scrape using `got-scraping` so it still returns results
|
|
174
178
|
instead of erroring. Fallback results set `model` to `"scrape:duckduckgo"` and
|
|
175
179
|
carry the substance in `citations` (there is no model synthesizing an answer,
|
|
176
|
-
so `answer` is a lead-in over the top results). It is enabled by default
|
|
180
|
+
so `answer` is a lead-in over the top results). It is enabled by default. Set
|
|
177
181
|
`scrapeFallback: false` (or `WEB_SEARCH_SCRAPE_FALLBACK=0`) to require a native
|
|
178
182
|
model and error when none is deployed.
|
|
179
183
|
|
|
180
|
-
The
|
|
181
|
-
OpenAI GPT uses the Responses API `{"tools":[{"type":"web_search"}]}
|
|
184
|
+
The plugin selects the provider request fragment from the resolved model:
|
|
185
|
+
OpenAI GPT uses the Responses API `{"tools":[{"type":"web_search"}]}`. Gemini
|
|
182
186
|
uses the top-level Chat Completions field `{"google_search":{}}`. Override or extend that map per provider with
|
|
183
187
|
the `webSearchTools` setting (env `WEB_SEARCH_TOOLS`, JSON) as the platform
|
|
184
188
|
evolves:
|
|
@@ -190,7 +194,7 @@ plugin.webSearch({
|
|
|
190
194
|
});
|
|
191
195
|
```
|
|
192
196
|
|
|
193
|
-
## Search
|
|
197
|
+
## Search and fetch without an agent
|
|
194
198
|
|
|
195
199
|
```ts
|
|
196
200
|
import { search, fetch, runtime, tool } from "@dbx-tools/appkit-web-search";
|
|
@@ -217,7 +221,7 @@ const page = await fetch.runWebFetch({ url: result.citations[0]!.url, format: "t
|
|
|
217
221
|
|
|
218
222
|
Use direct calls for operational lookups, tests, or admin flows where a model is
|
|
219
223
|
not involved. Standalone Mastra tool factories likewise accept `runtime` or
|
|
220
|
-
`config
|
|
224
|
+
`config`. They never discover or attach to a process-global plugin:
|
|
221
225
|
|
|
222
226
|
```ts
|
|
223
227
|
const webRuntime = runtime.createWebSearchRuntime({
|
|
@@ -233,7 +237,7 @@ const tools = {
|
|
|
233
237
|
Pass the runtime explicitly or provide the executor to
|
|
234
238
|
`createWebSearchRuntime`.
|
|
235
239
|
|
|
236
|
-
## Restrict
|
|
240
|
+
## Restrict reachable URLs
|
|
237
241
|
|
|
238
242
|
```ts
|
|
239
243
|
import { allowlist } from "@dbx-tools/appkit-web-search";
|
|
@@ -246,9 +250,9 @@ list.allows("https://docs.databricks.com/aws/en/index.html"); // true
|
|
|
246
250
|
list.allows("https://evil.example.com/"); // false
|
|
247
251
|
```
|
|
248
252
|
|
|
249
|
-
Each entry is a glob compiled by [
|
|
250
|
-
A host entry (no path,
|
|
251
|
-
URL's hostname, and a bare host also matches its subdomains
|
|
253
|
+
Each entry is a glob compiled by [@dbx-tools/path](../path)'s `match` matcher.
|
|
254
|
+
A host entry (no path, for example `databricks.com` or `*.databricks.com`) matches the
|
|
255
|
+
URL's hostname, and a bare host also matches its subdomains. A path entry
|
|
252
256
|
(`docs.example.com/api/**`) matches host + pathname. Enforcement is asymmetric by
|
|
253
257
|
design: `web_search` citations are silently filtered to the permitted set (the
|
|
254
258
|
model never surfaces a source it then can't fetch), while an explicit `web_fetch`
|
|
@@ -277,10 +281,10 @@ plugin.webSearch({ urlPolicy: "unrestricted" });
|
|
|
277
281
|
plugin.webSearch({ urlPolicy: "allowlist", allowedUrls: ["*.databricks.com"] });
|
|
278
282
|
```
|
|
279
283
|
|
|
280
|
-
## Gate
|
|
284
|
+
## Gate calls for approval
|
|
281
285
|
|
|
282
286
|
Both tools run without approval by default. Pass `approval` to a tool (or set it
|
|
283
|
-
plugin-wide) to require
|
|
287
|
+
plugin-wide) to require approval. `true` gates every call. A URL pattern (or
|
|
284
288
|
list) gates only calls whose URL matches - a `web_fetch` is evaluated precisely
|
|
285
289
|
against its target URL, while a `web_search` (whose result URLs are unknown
|
|
286
290
|
before the call) treats a pattern gate as "gate every call".
|
|
@@ -307,7 +311,7 @@ tool.webFetchTool({ config: { approval: ["*.internal.example.com"] } });
|
|
|
307
311
|
Every spelling normalizes to one of three modes - `{ mode: "none" }`,
|
|
308
312
|
`{ mode: "always" }`, `{ mode: "urls", patterns }` - which is what the resolved
|
|
309
313
|
config carries and the boot log reports. Approval uses the same glob syntax as
|
|
310
|
-
the
|
|
314
|
+
the allowlist, so the two policies read identically.
|
|
311
315
|
|
|
312
316
|
## Configuration
|
|
313
317
|
|
|
@@ -334,7 +338,7 @@ built-in default.
|
|
|
334
338
|
| `WEB_SEARCH_MODEL` | `model` (dedicated override, wins over the binding below) |
|
|
335
339
|
| `DATABRICKS_SERVING_ENDPOINT_NAME` | `model` (AppKit's Model Serving binding, treated as a preference) |
|
|
336
340
|
| `WEB_SEARCH_MODEL_FALLBACKS` | `modelFallbacks` (comma/space-separated) |
|
|
337
|
-
| `WEB_SEARCH_TOOLS` | `webSearchTools` (JSON object
|
|
341
|
+
| `WEB_SEARCH_TOOLS` | `webSearchTools` (JSON object. Any other value fails at boot.) |
|
|
338
342
|
| `WEB_SEARCH_FUZZY` | `modelFuzzyMatch` |
|
|
339
343
|
| `WEB_SEARCH_FUZZY_THRESHOLD` | `modelFuzzyThreshold` |
|
|
340
344
|
| `WEB_SEARCH_MAX_CITATIONS` | `maxCitations` |
|
|
@@ -355,10 +359,10 @@ TTLs and retry budgets and why each was chosen.
|
|
|
355
359
|
|
|
356
360
|
OpenAI Responses web searches stream native search, page-read, and completion
|
|
357
361
|
status through the Mastra tool writer. The common tool-session pill updates
|
|
358
|
-
while the search runs
|
|
362
|
+
while the search runs. The completed Responses payload remains the tool result.
|
|
359
363
|
|
|
360
364
|
Requirements: the native web-search tool runs on pay-per-token GPT / Gemini
|
|
361
|
-
serving endpoints with cross-region processing enabled
|
|
365
|
+
serving endpoints with cross-region processing enabled. It is unavailable on
|
|
362
366
|
provisioned throughput, for external models, or under HIPAA/BAA compliance. See
|
|
363
367
|
the [Databricks docs](https://docs.databricks.com/aws/en/machine-learning/model-serving/web-search).
|
|
364
368
|
|
|
@@ -381,9 +385,9 @@ the [Databricks docs](https://docs.databricks.com/aws/en/machine-learning/model-
|
|
|
381
385
|
and `executeRead()`.
|
|
382
386
|
- `config` - config types, JSON schema, `resolveWebSearchConfig()`, URL policy
|
|
383
387
|
and approval helpers.
|
|
384
|
-
- `allowlist` - URL
|
|
388
|
+
- `allowlist` - URL allowlist parsing/compiling on top of `@dbx-tools/path`.
|
|
385
389
|
- `schema` - zod tool contracts and inferred types.
|
|
386
390
|
|
|
387
|
-
Pair this package with [
|
|
388
|
-
the tools
|
|
389
|
-
|
|
391
|
+
Pair this package with [@dbx-tools/appkit-mastra](../appkit-mastra) to add
|
|
392
|
+
the tools to an agent and resolve its model through [@dbx-tools/model](../model).
|
|
393
|
+
Use [@dbx-tools/email](../email) for the sibling approval-gated tool pattern.
|
package/index.ts
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// Hand edits are overwritten on the next watch; this file is read-only.
|
|
4
4
|
|
|
5
5
|
export const PACKAGE_IDENTIFIER = "@dbx-tools/appkit-web-search";
|
|
6
|
-
export const PACKAGE_VERSION = "0.9.
|
|
6
|
+
export const PACKAGE_VERSION = "0.9.136";
|
|
7
7
|
export * as allowlist from "./src/allowlist.ts";
|
|
8
8
|
export * as config from "./src/config.ts";
|
|
9
9
|
export * as defaults from "./src/defaults.ts";
|
|
@@ -21,6 +21,7 @@ export { MODEL_ENV, SERVING_ENDPOINT_ENV, DEFAULT_MODEL_FALLBACKS, DEFAULT_MAX_C
|
|
|
21
21
|
export type { ApprovalGate, ApprovalPolicy, UrlPolicyMode, ModelSource, WebSearchPluginConfig, ResolvedWebSearchConfig } from "./src/config.ts";
|
|
22
22
|
export { SEARCH_CACHE_TTL_SECONDS, FETCH_CACHE_TTL_SECONDS, SERVING_RETRY_ATTEMPTS, WEB_RETRY_ATTEMPTS, webSearchExecuteDefaults, webFetchExecuteDefaults, scrapeSearchExecuteDefaults } from "./src/defaults.ts";
|
|
23
23
|
export type { WebSearchExecuteConfig, WebSearchExecutionSettings } from "./src/defaults.ts";
|
|
24
|
+
export type { WebAssetPageFetchRequest, WebImageFetchRequest, WebImageFetchResult } from "./src/fetch.ts";
|
|
24
25
|
export { WebSearchPlugin, webSearch } from "./src/plugin.ts";
|
|
25
26
|
export { WebSearchProviderSpecSchema, WEB_SEARCH_PROVIDERS } from "./src/provider.ts";
|
|
26
27
|
export type { WebSearchProvider, WebSearchProviderSpec } from "./src/provider.ts";
|
package/lib/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export declare const PACKAGE_IDENTIFIER = "@dbx-tools/appkit-web-search";
|
|
2
|
-
export declare const PACKAGE_VERSION = "0.9.
|
|
2
|
+
export declare const PACKAGE_VERSION = "0.9.136";
|
|
3
3
|
export * as allowlist from "./src/allowlist.ts";
|
|
4
4
|
export * as config from "./src/config.ts";
|
|
5
5
|
export * as defaults from "./src/defaults.ts";
|
|
@@ -17,6 +17,7 @@ export { MODEL_ENV, SERVING_ENDPOINT_ENV, DEFAULT_MODEL_FALLBACKS, DEFAULT_MAX_C
|
|
|
17
17
|
export type { ApprovalGate, ApprovalPolicy, UrlPolicyMode, ModelSource, WebSearchPluginConfig, ResolvedWebSearchConfig } from "./src/config.ts";
|
|
18
18
|
export { SEARCH_CACHE_TTL_SECONDS, FETCH_CACHE_TTL_SECONDS, SERVING_RETRY_ATTEMPTS, WEB_RETRY_ATTEMPTS, webSearchExecuteDefaults, webFetchExecuteDefaults, scrapeSearchExecuteDefaults } from "./src/defaults.ts";
|
|
19
19
|
export type { WebSearchExecuteConfig, WebSearchExecutionSettings } from "./src/defaults.ts";
|
|
20
|
+
export type { WebAssetPageFetchRequest, WebImageFetchRequest, WebImageFetchResult } from "./src/fetch.ts";
|
|
20
21
|
export { WebSearchPlugin, webSearch } from "./src/plugin.ts";
|
|
21
22
|
export { WebSearchProviderSpecSchema, WEB_SEARCH_PROVIDERS } from "./src/provider.ts";
|
|
22
23
|
export type { WebSearchProvider, WebSearchProviderSpec } from "./src/provider.ts";
|
package/lib/index.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Regenerated from the exporting modules in ./src.
|
|
3
3
|
// Hand edits are overwritten on the next watch; this file is read-only.
|
|
4
4
|
export const PACKAGE_IDENTIFIER = "@dbx-tools/appkit-web-search";
|
|
5
|
-
export const PACKAGE_VERSION = "0.9.
|
|
5
|
+
export const PACKAGE_VERSION = "0.9.136";
|
|
6
6
|
export * as allowlist from "./src/allowlist.js";
|
|
7
7
|
export * as config from "./src/config.js";
|
|
8
8
|
export * as defaults from "./src/defaults.js";
|
|
@@ -20,4 +20,4 @@ export { SEARCH_CACHE_TTL_SECONDS, FETCH_CACHE_TTL_SECONDS, SERVING_RETRY_ATTEMP
|
|
|
20
20
|
export { WebSearchPlugin, webSearch } from "./src/plugin.js";
|
|
21
21
|
export { WebSearchProviderSpecSchema, WEB_SEARCH_PROVIDERS } from "./src/provider.js";
|
|
22
22
|
export { WEB_SEARCH_TOOL_DESCRIPTION, WEB_FETCH_TOOL_DESCRIPTION, webSearchRequestSchema, webSearchToolRequestSchema, webSearchCitationSchema, webSearchResultSchema, webFetchRequestSchema, webFetchResultSchema } from "./src/schema.js";
|
|
23
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
23
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9pbmRleC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSwyQ0FBMkM7QUFDM0MsbURBQW1EO0FBQ25ELHdFQUF3RTtBQUV4RSxNQUFNLENBQUMsTUFBTSxrQkFBa0IsR0FBRyw4QkFBOEIsQ0FBQztBQUNqRSxNQUFNLENBQUMsTUFBTSxlQUFlLEdBQUcsU0FBUyxDQUFDO0FBQ3pDLE9BQU8sS0FBSyxTQUFTLE1BQU0sb0JBQW9CLENBQUM7QUFDaEQsT0FBTyxLQUFLLE1BQU0sTUFBTSxpQkFBaUIsQ0FBQztBQUMxQyxPQUFPLEtBQUssUUFBUSxNQUFNLG1CQUFtQixDQUFDO0FBQzlDLE9BQU8sS0FBSyxLQUFLLE1BQU0sZ0JBQWdCLENBQUM7QUFDeEMsT0FBTyxLQUFLLFFBQVEsTUFBTSxvQkFBb0IsQ0FBQztBQUMvQyxPQUFPLEtBQUssTUFBTSxNQUFNLGlCQUFpQixDQUFDO0FBQzFDLE9BQU8sS0FBSyxRQUFRLE1BQU0sbUJBQW1CLENBQUM7QUFDOUMsT0FBTyxLQUFLLE9BQU8sTUFBTSxrQkFBa0IsQ0FBQztBQUM1QyxPQUFPLEtBQUssTUFBTSxNQUFNLGlCQUFpQixDQUFDO0FBQzFDLE9BQU8sS0FBSyxNQUFNLE1BQU0saUJBQWlCLENBQUM7QUFDMUMsT0FBTyxLQUFLLE1BQU0sTUFBTSxpQkFBaUIsQ0FBQztBQUMxQyxPQUFPLEtBQUssSUFBSSxNQUFNLGVBQWUsQ0FBQztBQUV0QyxPQUFPLEVBQUUsU0FBUyxFQUFFLG9CQUFvQixFQUFFLHVCQUF1QixFQUFFLHFCQUFxQixFQUFFLHdCQUF3QixFQUFFLGtCQUFrQixFQUFFLHdCQUF3QixFQUFFLE1BQU0saUJBQWlCLENBQUM7QUFFMUwsT0FBTyxFQUFFLHdCQUF3QixFQUFFLHVCQUF1QixFQUFFLHNCQUFzQixFQUFFLGtCQUFrQixFQUFFLHdCQUF3QixFQUFFLHVCQUF1QixFQUFFLDJCQUEyQixFQUFFLE1BQU0sbUJBQW1CLENBQUM7QUFHbE4sT0FBTyxFQUFFLGVBQWUsRUFBRSxTQUFTLEVBQUUsTUFBTSxpQkFBaUIsQ0FBQztBQUM3RCxPQUFPLEVBQUUsMkJBQTJCLEVBQUUsb0JBQW9CLEVBQUUsTUFBTSxtQkFBbUIsQ0FBQztBQUd0RixPQUFPLEVBQUUsMkJBQTJCLEVBQUUsMEJBQTBCLEVBQUUsc0JBQXNCLEVBQUUsMEJBQTBCLEVBQUUsdUJBQXVCLEVBQUUscUJBQXFCLEVBQUUscUJBQXFCLEVBQUUsb0JBQW9CLEVBQUUsTUFBTSxpQkFBaUIsQ0FBQyIsInNvdXJjZXNDb250ZW50IjpbIi8vIEdFTkVSQVRFRCBieSBwcm9qZW4gd2F0Y2ggLSBETyBOT1QgRURJVC5cbi8vIFJlZ2VuZXJhdGVkIGZyb20gdGhlIGV4cG9ydGluZyBtb2R1bGVzIGluIC4vc3JjLlxuLy8gSGFuZCBlZGl0cyBhcmUgb3ZlcndyaXR0ZW4gb24gdGhlIG5leHQgd2F0Y2g7IHRoaXMgZmlsZSBpcyByZWFkLW9ubHkuXG5cbmV4cG9ydCBjb25zdCBQQUNLQUdFX0lERU5USUZJRVIgPSBcIkBkYngtdG9vbHMvYXBwa2l0LXdlYi1zZWFyY2hcIjtcbmV4cG9ydCBjb25zdCBQQUNLQUdFX1ZFUlNJT04gPSBcIjAuOS4xMzZcIjtcbmV4cG9ydCAqIGFzIGFsbG93bGlzdCBmcm9tIFwiLi9zcmMvYWxsb3dsaXN0LnRzXCI7XG5leHBvcnQgKiBhcyBjb25maWcgZnJvbSBcIi4vc3JjL2NvbmZpZy50c1wiO1xuZXhwb3J0ICogYXMgZGVmYXVsdHMgZnJvbSBcIi4vc3JjL2RlZmF1bHRzLnRzXCI7XG5leHBvcnQgKiBhcyBmZXRjaCBmcm9tIFwiLi9zcmMvZmV0Y2gudHNcIjtcbmV4cG9ydCAqIGFzIGh0bWxUZXh0IGZyb20gXCIuL3NyYy9odG1sLXRleHQudHNcIjtcbmV4cG9ydCAqIGFzIHBsdWdpbiBmcm9tIFwiLi9zcmMvcGx1Z2luLnRzXCI7XG5leHBvcnQgKiBhcyBwcm92aWRlciBmcm9tIFwiLi9zcmMvcHJvdmlkZXIudHNcIjtcbmV4cG9ydCAqIGFzIHJ1bnRpbWUgZnJvbSBcIi4vc3JjL3J1bnRpbWUudHNcIjtcbmV4cG9ydCAqIGFzIHNjaGVtYSBmcm9tIFwiLi9zcmMvc2NoZW1hLnRzXCI7XG5leHBvcnQgKiBhcyBzY3JhcGUgZnJvbSBcIi4vc3JjL3NjcmFwZS50c1wiO1xuZXhwb3J0ICogYXMgc2VhcmNoIGZyb20gXCIuL3NyYy9zZWFyY2gudHNcIjtcbmV4cG9ydCAqIGFzIHRvb2wgZnJvbSBcIi4vc3JjL3Rvb2wudHNcIjtcbmV4cG9ydCB0eXBlIHsgVXJsQWxsb3dMaXN0IH0gZnJvbSBcIi4vc3JjL2FsbG93bGlzdC50c1wiO1xuZXhwb3J0IHsgTU9ERUxfRU5WLCBTRVJWSU5HX0VORFBPSU5UX0VOViwgREVGQVVMVF9NT0RFTF9GQUxMQkFDS1MsIERFRkFVTFRfTUFYX0NJVEFUSU9OUywgREVGQVVMVF9GRVRDSF9NQVhfTEVOR1RILCBERUZBVUxUX1RJTUVPVVRfTVMsIFdFQl9TRUFSQ0hfQ09ORklHX1NDSEVNQSB9IGZyb20gXCIuL3NyYy9jb25maWcudHNcIjtcbmV4cG9ydCB0eXBlIHsgQXBwcm92YWxHYXRlLCBBcHByb3ZhbFBvbGljeSwgVXJsUG9saWN5TW9kZSwgTW9kZWxTb3VyY2UsIFdlYlNlYXJjaFBsdWdpbkNvbmZpZywgUmVzb2x2ZWRXZWJTZWFyY2hDb25maWcgfSBmcm9tIFwiLi9zcmMvY29uZmlnLnRzXCI7XG5leHBvcnQgeyBTRUFSQ0hfQ0FDSEVfVFRMX1NFQ09ORFMsIEZFVENIX0NBQ0hFX1RUTF9TRUNPTkRTLCBTRVJWSU5HX1JFVFJZX0FUVEVNUFRTLCBXRUJfUkVUUllfQVRURU1QVFMsIHdlYlNlYXJjaEV4ZWN1dGVEZWZhdWx0cywgd2ViRmV0Y2hFeGVjdXRlRGVmYXVsdHMsIHNjcmFwZVNlYXJjaEV4ZWN1dGVEZWZhdWx0cyB9IGZyb20gXCIuL3NyYy9kZWZhdWx0cy50c1wiO1xuZXhwb3J0IHR5cGUgeyBXZWJTZWFyY2hFeGVjdXRlQ29uZmlnLCBXZWJTZWFyY2hFeGVjdXRpb25TZXR0aW5ncyB9IGZyb20gXCIuL3NyYy9kZWZhdWx0cy50c1wiO1xuZXhwb3J0IHR5cGUgeyBXZWJBc3NldFBhZ2VGZXRjaFJlcXVlc3QsIFdlYkltYWdlRmV0Y2hSZXF1ZXN0LCBXZWJJbWFnZUZldGNoUmVzdWx0IH0gZnJvbSBcIi4vc3JjL2ZldGNoLnRzXCI7XG5leHBvcnQgeyBXZWJTZWFyY2hQbHVnaW4sIHdlYlNlYXJjaCB9IGZyb20gXCIuL3NyYy9wbHVnaW4udHNcIjtcbmV4cG9ydCB7IFdlYlNlYXJjaFByb3ZpZGVyU3BlY1NjaGVtYSwgV0VCX1NFQVJDSF9QUk9WSURFUlMgfSBmcm9tIFwiLi9zcmMvcHJvdmlkZXIudHNcIjtcbmV4cG9ydCB0eXBlIHsgV2ViU2VhcmNoUHJvdmlkZXIsIFdlYlNlYXJjaFByb3ZpZGVyU3BlYyB9IGZyb20gXCIuL3NyYy9wcm92aWRlci50c1wiO1xuZXhwb3J0IHR5cGUgeyBXZWJTZWFyY2hFeGVjdXRvciwgV2ViU2VhcmNoUnVudGltZSwgV2ViU2VhcmNoUnVudGltZUlucHV0IH0gZnJvbSBcIi4vc3JjL3J1bnRpbWUudHNcIjtcbmV4cG9ydCB7IFdFQl9TRUFSQ0hfVE9PTF9ERVNDUklQVElPTiwgV0VCX0ZFVENIX1RPT0xfREVTQ1JJUFRJT04sIHdlYlNlYXJjaFJlcXVlc3RTY2hlbWEsIHdlYlNlYXJjaFRvb2xSZXF1ZXN0U2NoZW1hLCB3ZWJTZWFyY2hDaXRhdGlvblNjaGVtYSwgd2ViU2VhcmNoUmVzdWx0U2NoZW1hLCB3ZWJGZXRjaFJlcXVlc3RTY2hlbWEsIHdlYkZldGNoUmVzdWx0U2NoZW1hIH0gZnJvbSBcIi4vc3JjL3NjaGVtYS50c1wiO1xuZXhwb3J0IHR5cGUgeyBXZWJTZWFyY2hSZXF1ZXN0LCBXZWJTZWFyY2hUb29sUmVxdWVzdCwgV2ViU2VhcmNoQ2l0YXRpb24sIFdlYlNlYXJjaFJlc3VsdCwgV2ViRmV0Y2hSZXF1ZXN0LCBXZWJGZXRjaFJlc3VsdCB9IGZyb20gXCIuL3NyYy9zY2hlbWEudHNcIjtcbmV4cG9ydCB0eXBlIHsgV2ViU2VhcmNoQ29udGV4dCwgV2ViU2VhcmNoTW9kZWxTZWxlY3Rpb25PcHRpb25zIH0gZnJvbSBcIi4vc3JjL3NlYXJjaC50c1wiO1xuZXhwb3J0IHR5cGUgeyBXZWJTZWFyY2hUb29sT3B0aW9ucyB9IGZyb20gXCIuL3NyYy90b29sLnRzXCI7XG4iXX0=
|
package/lib/src/fetch.d.ts
CHANGED
|
@@ -16,6 +16,29 @@
|
|
|
16
16
|
*/
|
|
17
17
|
import { type WebSearchRuntimeInput } from "./runtime.ts";
|
|
18
18
|
import type { WebFetchRequest, WebFetchResult } from "./schema.ts";
|
|
19
|
+
/** Input for a policy-checked HTML fetch used by in-process asset discovery. */
|
|
20
|
+
export interface WebAssetPageFetchRequest {
|
|
21
|
+
/** Public HTTPS page URL. */
|
|
22
|
+
url: string;
|
|
23
|
+
/** Maximum returned markup length. Defaults to 1.5 million characters. */
|
|
24
|
+
maxLength?: number;
|
|
25
|
+
}
|
|
26
|
+
/** Input for a policy-checked binary image fetch. */
|
|
27
|
+
export interface WebImageFetchRequest {
|
|
28
|
+
/** Public HTTPS image URL. */
|
|
29
|
+
url: string;
|
|
30
|
+
/** Maximum accepted response size. Defaults to 5 MiB. */
|
|
31
|
+
maxBytes?: number;
|
|
32
|
+
}
|
|
33
|
+
/** Binary image fetched through the web-search runtime's URL policy. */
|
|
34
|
+
export interface WebImageFetchResult {
|
|
35
|
+
/** Final URL after redirects. */
|
|
36
|
+
url: string;
|
|
37
|
+
/** Image media type returned by the server. */
|
|
38
|
+
contentType: string;
|
|
39
|
+
/** Fetched image bytes. */
|
|
40
|
+
data: Uint8Array;
|
|
41
|
+
}
|
|
19
42
|
/**
|
|
20
43
|
* Fetch a single URL and return its content in the requested format.
|
|
21
44
|
*
|
|
@@ -25,3 +48,11 @@ import type { WebFetchRequest, WebFetchResult } from "./schema.ts";
|
|
|
25
48
|
* `signal` cancels the in-flight fetch.
|
|
26
49
|
*/
|
|
27
50
|
export declare function runWebFetch(request: WebFetchRequest, runtimeOrConfig: WebSearchRuntimeInput, signal?: AbortSignal): Promise<WebFetchResult>;
|
|
51
|
+
/**
|
|
52
|
+
* Fetch bounded raw markup for deterministic in-process asset discovery.
|
|
53
|
+
*
|
|
54
|
+
* This does not widen the model-facing `web_fetch` cap.
|
|
55
|
+
*/
|
|
56
|
+
export declare function runWebAssetPageFetch(request: WebAssetPageFetchRequest, runtimeOrConfig: WebSearchRuntimeInput, signal?: AbortSignal): Promise<WebFetchResult>;
|
|
57
|
+
/** Fetch one public image while preserving URL policy, timeout, and redirect checks. */
|
|
58
|
+
export declare function runWebImageFetch(request: WebImageFetchRequest, runtimeOrConfig: WebSearchRuntimeInput, signal?: AbortSignal): Promise<WebImageFetchResult>;
|
package/lib/src/fetch.js
CHANGED
|
@@ -19,8 +19,10 @@ import { gotScraping } from "got-scraping";
|
|
|
19
19
|
import { assertFetchUrlAllowed } from "./_fetch-url.js";
|
|
20
20
|
import { toCallSettings, webFetchExecuteDefaults } from "./defaults.js";
|
|
21
21
|
import { decodeHtmlEntities, htmlToText } from "./html-text.js";
|
|
22
|
-
import { executeRead, toWebSearchRuntime } from "./runtime.js";
|
|
22
|
+
import { executeRead, toWebSearchRuntime, } from "./runtime.js";
|
|
23
23
|
const logger = log.logger("web-search/fetch");
|
|
24
|
+
const DEFAULT_IMAGE_MAX_BYTES = 5 * 1024 * 1024;
|
|
25
|
+
const DEFAULT_ASSET_PAGE_MAX_LENGTH = 1_500_000;
|
|
24
26
|
/** Pull the <title> text out of an HTML document, when present. */
|
|
25
27
|
function extractTitle(html) {
|
|
26
28
|
const match = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(html);
|
|
@@ -33,29 +35,18 @@ function truncate(text, max) {
|
|
|
33
35
|
return { content: text, truncated: false };
|
|
34
36
|
return { content: text.slice(0, max), truncated: true };
|
|
35
37
|
}
|
|
36
|
-
/**
|
|
37
|
-
|
|
38
|
-
*
|
|
39
|
-
* Throws when the URL is not permitted by the allow-list (the visible,
|
|
40
|
-
* correctable failure the design calls for on the fetch path). The request's
|
|
41
|
-
* `maxLength` narrows (never widens) the plugin's `fetchMaxLength` cap.
|
|
42
|
-
* `signal` cancels the in-flight fetch.
|
|
43
|
-
*/
|
|
44
|
-
export async function runWebFetch(request, runtimeOrConfig, signal) {
|
|
45
|
-
const runtime = toWebSearchRuntime(runtimeOrConfig);
|
|
38
|
+
/** Fetch and cache one raw page before caller-specific output reduction. */
|
|
39
|
+
async function fetchPage(requestUrl, runtime, signal) {
|
|
46
40
|
const { config } = runtime;
|
|
47
|
-
await assertFetchUrlAllowed(
|
|
48
|
-
const cap = Math.min(request.maxLength ?? config.fetchMaxLength, config.fetchMaxLength);
|
|
49
|
-
// The response is cached before the format reduction, so the same page read
|
|
50
|
-
// as text and as html shares one network round trip.
|
|
41
|
+
await assertFetchUrlAllowed(requestUrl, config.allowList);
|
|
51
42
|
const page = await executeRead(runtime, "page-fetch", toCallSettings(webFetchExecuteDefaults, config.timeoutMs, [
|
|
52
43
|
"web-search",
|
|
53
44
|
"fetch",
|
|
54
|
-
|
|
45
|
+
requestUrl,
|
|
55
46
|
...config.allowList.patterns,
|
|
56
47
|
]), async (executeSignal) => {
|
|
57
48
|
const response = await gotScraping({
|
|
58
|
-
url:
|
|
49
|
+
url: requestUrl,
|
|
59
50
|
// Bun's node:http2 reports default-port origins with :443 while
|
|
60
51
|
// http2-wrapper compares them against URL-normalized origins.
|
|
61
52
|
http2: false,
|
|
@@ -74,13 +65,28 @@ export async function runWebFetch(request, runtimeOrConfig, signal) {
|
|
|
74
65
|
...(executeSignal ? { signal: executeSignal } : {}),
|
|
75
66
|
});
|
|
76
67
|
return {
|
|
77
|
-
url: response.url ??
|
|
68
|
+
url: response.url ?? requestUrl,
|
|
78
69
|
statusCode: response.statusCode,
|
|
79
70
|
contentType: response.headers["content-type"],
|
|
80
71
|
body: typeof response.body === "string" ? response.body : String(response.body ?? ""),
|
|
81
72
|
};
|
|
82
73
|
}, signal);
|
|
83
74
|
await assertFetchUrlAllowed(page.url, config.allowList);
|
|
75
|
+
return page;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Fetch a single URL and return its content in the requested format.
|
|
79
|
+
*
|
|
80
|
+
* Throws when the URL is not permitted by the allow-list (the visible,
|
|
81
|
+
* correctable failure the design calls for on the fetch path). The request's
|
|
82
|
+
* `maxLength` narrows (never widens) the plugin's `fetchMaxLength` cap.
|
|
83
|
+
* `signal` cancels the in-flight fetch.
|
|
84
|
+
*/
|
|
85
|
+
export async function runWebFetch(request, runtimeOrConfig, signal) {
|
|
86
|
+
const runtime = toWebSearchRuntime(runtimeOrConfig);
|
|
87
|
+
const { config } = runtime;
|
|
88
|
+
const cap = Math.min(request.maxLength ?? config.fetchMaxLength, config.fetchMaxLength);
|
|
89
|
+
const page = await fetchPage(request.url, runtime, signal);
|
|
84
90
|
const isHtml = !page.contentType || /html|xml/i.test(page.contentType);
|
|
85
91
|
const rawContent = request.format === "html" || !isHtml ? page.body : htmlToText(page.body);
|
|
86
92
|
const { content, truncated } = truncate(rawContent, cap);
|
|
@@ -101,4 +107,72 @@ export async function runWebFetch(request, runtimeOrConfig, signal) {
|
|
|
101
107
|
truncated,
|
|
102
108
|
};
|
|
103
109
|
}
|
|
104
|
-
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmV0Y2guanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvZmV0Y2gudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUE7Ozs7Ozs7Ozs7Ozs7OztHQWVHO0FBRUgsT0FBTyxFQUFFLEdBQUcsRUFBRSxNQUFNLHdCQUF3QixDQUFDO0FBQzdDLE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSxjQUFjLENBQUM7QUFDM0MsT0FBTyxFQUFFLHFCQUFxQixFQUFFLE1BQU0saUJBQWlCLENBQUM7QUFDeEQsT0FBTyxFQUFFLGNBQWMsRUFBRSx1QkFBdUIsRUFBRSxNQUFNLGVBQWUsQ0FBQztBQUN4RSxPQUFPLEVBQUUsa0JBQWtCLEVBQUUsVUFBVSxFQUFFLE1BQU0sZ0JBQWdCLENBQUM7QUFDaEUsT0FBTyxFQUFFLFdBQVcsRUFBRSxrQkFBa0IsRUFBOEIsTUFBTSxjQUFjLENBQUM7QUFHM0YsTUFBTSxNQUFNLEdBQUcsR0FBRyxDQUFDLE1BQU0sQ0FBQyxrQkFBa0IsQ0FBQyxDQUFDO0FBVTlDLG1FQUFtRTtBQUNuRSxTQUFTLFlBQVksQ0FBQyxJQUFZO0lBQ2hDLE1BQU0sS0FBSyxHQUFHLGtDQUFrQyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQztJQUM1RCxNQUFNLEtBQUssR0FBRyxLQUFLLEVBQUUsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsa0JBQWtCLENBQUMsS0FBSyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsSUFBSSxFQUFFLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQztJQUNwRSxPQUFPLEtBQUssQ0FBQyxNQUFNLEdBQUcsQ0FBQyxDQUFDLENBQUMsQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDLFNBQVMsQ0FBQztBQUM5QyxDQUFDO0FBRUQsb0VBQW9FO0FBQ3BFLFNBQVMsUUFBUSxDQUFDLElBQVksRUFBRSxHQUFXO0lBQ3pDLElBQUksSUFBSSxDQUFDLE1BQU0sSUFBSSxHQUFHO1FBQUUsT0FBTyxFQUFFLE9BQU8sRUFBRSxJQUFJLEVBQUUsU0FBUyxFQUFFLEtBQUssRUFBRSxDQUFDO0lBQ25FLE9BQU8sRUFBRSxPQUFPLEVBQUUsSUFBSSxDQUFDLEtBQUssQ0FBQyxDQUFDLEVBQUUsR0FBRyxDQUFDLEVBQUUsU0FBUyxFQUFFLElBQUksRUFBRSxDQUFDO0FBQzFELENBQUM7QUFFRDs7Ozs7OztHQU9HO0FBQ0gsTUFBTSxDQUFDLEtBQUssVUFBVSxXQUFXLENBQy9CLE9BQXdCLEVBQ3hCLGVBQXNDLEVBQ3RDLE1BQW9CO0lBRXBCLE1BQU0sT0FBTyxHQUFHLGtCQUFrQixDQUFDLGVBQWUsQ0FBQyxDQUFDO0lBQ3BELE1BQU0sRUFBRSxNQUFNLEVBQUUsR0FBRyxPQUFPLENBQUM7SUFDM0IsTUFBTSxxQkFBcUIsQ0FBQyxPQUFPLENBQUMsR0FBRyxFQUFFLE1BQU0sQ0FBQyxTQUFTLENBQUMsQ0FBQztJQUMzRCxNQUFNLEdBQUcsR0FBRyxJQUFJLENBQUMsR0FBRyxDQUFDLE9BQU8sQ0FBQyxTQUFTLElBQUksTUFBTSxDQUFDLGNBQWMsRUFBRSxNQUFNLENBQUMsY0FBYyxDQUFDLENBQUM7SUFFeEYsNEVBQTRFO0lBQzVFLHFEQUFxRDtJQUNyRCxNQUFNLElBQUksR0FBRyxNQUFNLFdBQVcsQ0FDNUIsT0FBTyxFQUNQLFlBQVksRUFDWixjQUFjLENBQUMsdUJBQXVCLEVBQUUsTUFBTSxDQUFDLFNBQVMsRUFBRTtRQUN4RCxZQUFZO1FBQ1osT0FBTztRQUNQLE9BQU8sQ0FBQyxHQUFHO1FBQ1gsR0FBRyxNQUFNLENBQUMsU0FBUyxDQUFDLFFBQVE7S0FDN0IsQ0FBQyxFQUNGLEtBQUssRUFBRSxhQUFhLEVBQXdCLEVBQUU7UUFDNUMsTUFBTSxRQUFRLEdBQUcsTUFBTSxXQUFXLENBQUM7WUFDakMsR0FBRyxFQUFFLE9BQU8sQ0FBQyxHQUFHO1lBQ2hCLGdFQUFnRTtZQUNoRSw4REFBOEQ7WUFDOUQsS0FBSyxFQUFFLEtBQUs7WUFDWixvRUFBb0U7WUFDcEUsbUVBQW1FO1lBQ25FLE9BQU8sRUFBRSxFQUFFLE9BQU8sRUFBRSxNQUFNLENBQUMsU0FBUyxFQUFFO1lBQ3RDLGVBQWUsRUFBRSxLQUFLO1lBQ3RCLGNBQWMsRUFBRSxJQUFJO1lBQ3BCLEtBQUssRUFBRTtnQkFDTCxhQUFhLEVBQUU7b0JBQ2IsS0FBSyxFQUFFLE9BQU8sRUFBRSxFQUFFO3dCQUNoQixNQUFNLHFCQUFxQixDQUFDLE1BQU0sQ0FBQyxPQUFPLENBQUMsR0FBRyxDQUFDLEVBQUUsTUFBTSxDQUFDLFNBQVMsQ0FBQyxDQUFDO29CQUNyRSxDQUFDO2lCQUNGO2FBQ0Y7WUFDRCxHQUFHLENBQUMsYUFBYSxDQUFDLENBQUMsQ0FBQyxFQUFFLE1BQU0sRUFBRSxhQUFhLEVBQUUsQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDO1NBQ3BELENBQUMsQ0FBQztRQUNILE9BQU87WUFDTCxHQUFHLEVBQUUsUUFBUSxDQUFDLEdBQUcsSUFBSSxPQUFPLENBQUMsR0FBRztZQUNoQyxVQUFVLEVBQUUsUUFBUSxDQUFDLFVBQVU7WUFDL0IsV0FBVyxFQUFFLFFBQVEsQ0FBQyxPQUFPLENBQUMsY0FBYyxDQUFDO1lBQzdDLElBQUksRUFBRSxPQUFPLFFBQVEsQ0FBQyxJQUFJLEtBQUssUUFBUSxDQUFDLENBQUMsQ0FBQyxRQUFRLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxNQUFNLENBQUMsUUFBUSxDQUFDLElBQUksSUFBSSxFQUFFLENBQUM7U0FDdEYsQ0FBQztJQUNKLENBQUMsRUFDRCxNQUFNLENBQ1AsQ0FBQztJQUNGLE1BQU0scUJBQXFCLENBQUMsSUFBSSxDQUFDLEdBQUcsRUFBRSxNQUFNLENBQUMsU0FBUyxDQUFDLENBQUM7SUFFeEQsTUFBTSxNQUFNLEdBQUcsQ0FBQyxJQUFJLENBQUMsV0FBVyxJQUFJLFdBQVcsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLFdBQVcsQ0FBQyxDQUFDO0lBQ3ZFLE1BQU0sVUFBVSxHQUFHLE9BQU8sQ0FBQyxNQUFNLEtBQUssTUFBTSxJQUFJLENBQUMsTUFBTSxDQUFDLENBQUMsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxVQUFVLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDO0lBQzVGLE1BQU0sRUFBRSxPQUFPLEVBQUUsU0FBUyxFQUFFLEdBQUcsUUFBUSxDQUFDLFVBQVUsRUFBRSxHQUFHLENBQUMsQ0FBQztJQUN6RCxNQUFNLEtBQUssR0FBRyxNQUFNLENBQUMsQ0FBQyxDQUFDLFlBQVksQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxDQUFDLFNBQVMsQ0FBQztJQUUzRCxNQUFNLENBQUMsS0FBSyxDQUFDLFNBQVMsRUFBRTtRQUN0QixHQUFHLEVBQUUsSUFBSSxDQUFDLEdBQUc7UUFDYixNQUFNLEVBQUUsSUFBSSxDQUFDLFVBQVU7UUFDdkIsS0FBSyxFQUFFLElBQUksQ0FBQyxJQUFJLENBQUMsTUFBTTtRQUN2QixRQUFRLEVBQUUsT0FBTyxDQUFDLE1BQU07UUFDeEIsR0FBRyxDQUFDLFNBQVMsQ0FBQyxDQUFDLENBQUMsRUFBRSxTQUFTLEVBQUUsSUFBSSxFQUFFLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQztLQUMxQyxDQUFDLENBQUM7SUFFSCxPQUFPO1FBQ0wsR0FBRyxFQUFFLElBQUksQ0FBQyxHQUFHO1FBQ2IsTUFBTSxFQUFFLElBQUksQ0FBQyxVQUFVO1FBQ3ZCLEdBQUcsQ0FBQyxJQUFJLENBQUMsV0FBVyxDQUFDLENBQUMsQ0FBQyxFQUFFLFdBQVcsRUFBRSxJQUFJLENBQUMsV0FBVyxFQUFFLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQztRQUM5RCxHQUFHLENBQUMsS0FBSyxDQUFDLENBQUMsQ0FBQyxFQUFFLEtBQUssRUFBRSxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUM7UUFDM0IsT0FBTztRQUNQLFNBQVM7S0FDVixDQUFDO0FBQ0osQ0FBQyIsInNvdXJjZXNDb250ZW50IjpbIi8qKlxuICogUGFnZSBmZXRjaGluZyBvdmVyIHtAbGluayBodHRwczovL3d3dy5ucG1qcy5jb20vcGFja2FnZS9nb3Qtc2NyYXBpbmcgfCBnb3Qtc2NyYXBpbmd9XG4gKiAtIGEgYGdvdGAgd3JhcHBlciB0aGF0IGdlbmVyYXRlcyBicm93c2VyLWxpa2UgVExTICsgaGVhZGVyIGZpbmdlcnByaW50cywgc29cbiAqIGZldGNoZXMgc3Vydml2ZSB0aGUgYm90IHdhbGxzIGEgcGxhaW4gYGZldGNoYCB0cmlwcy4ge0BsaW5rIHJ1bldlYkZldGNofVxuICogZW5mb3JjZXMgdGhlIFVSTCBhbGxvdy1saXN0IChhbiBleHBsaWNpdCBmZXRjaCBvZiBhIGRpc2FsbG93ZWQgVVJMIGlzXG4gKiByZWZ1c2VkLCBub3Qgc2lsZW50bHkgZW1wdGllZCksIGZldGNoZXMgd2l0aCB0aGUgcGx1Z2luJ3MgdGltZW91dCwgYW5kXG4gKiByZXR1cm5zIGVpdGhlciB0aGUgcmF3IEhUTUwgb3IgYSByZWFkYWJsZSBwbGFpbi10ZXh0IHJlZHVjdGlvbiwgY2FwcGVkIGF0XG4gKiB0aGUgY29uZmlndXJlZCBsZW5ndGguXG4gKlxuICogVGhlIEhUTUwtdG8tdGV4dCByZWR1Y3Rpb24gaXMgZGVsaWJlcmF0ZWx5IGRlcGVuZGVuY3ktZnJlZSAoc3RyaXBcbiAqIHNjcmlwdC9zdHlsZSwgdW53cmFwIHRhZ3MsIGRlY29kZSBhIGhhbmRmdWwgb2YgZW50aXRpZXMsIGNvbGxhcHNlXG4gKiB3aGl0ZXNwYWNlKTogZ29vZCBlbm91Z2ggdG8gZmVlZCBhIG1vZGVsLCBhbmQgaXQga2VlcHMgdGhlIGFkZC1vbidzXG4gKiBkZXBlbmRlbmN5IHN1cmZhY2UgdG8gdGhlIHR3byBsaWJyYXJpZXMgdGhlIHRhc2sgY2FsbGVkIGZvci5cbiAqXG4gKiBAbW9kdWxlXG4gKi9cblxuaW1wb3J0IHsgbG9nIH0gZnJvbSBcIkBkYngtdG9vbHMvc2hhcmVkLWNvcmVcIjtcbmltcG9ydCB7IGdvdFNjcmFwaW5nIH0gZnJvbSBcImdvdC1zY3JhcGluZ1wiO1xuaW1wb3J0IHsgYXNzZXJ0RmV0Y2hVcmxBbGxvd2VkIH0gZnJvbSBcIi4vX2ZldGNoLXVybC50c1wiO1xuaW1wb3J0IHsgdG9DYWxsU2V0dGluZ3MsIHdlYkZldGNoRXhlY3V0ZURlZmF1bHRzIH0gZnJvbSBcIi4vZGVmYXVsdHMudHNcIjtcbmltcG9ydCB7IGRlY29kZUh0bWxFbnRpdGllcywgaHRtbFRvVGV4dCB9IGZyb20gXCIuL2h0bWwtdGV4dC50c1wiO1xuaW1wb3J0IHsgZXhlY3V0ZVJlYWQsIHRvV2ViU2VhcmNoUnVudGltZSwgdHlwZSBXZWJTZWFyY2hSdW50aW1lSW5wdXQgfSBmcm9tIFwiLi9ydW50aW1lLnRzXCI7XG5pbXBvcnQgdHlwZSB7IFdlYkZldGNoUmVxdWVzdCwgV2ViRmV0Y2hSZXN1bHQgfSBmcm9tIFwiLi9zY2hlbWEudHNcIjtcblxuY29uc3QgbG9nZ2VyID0gbG9nLmxvZ2dlcihcIndlYi1zZWFyY2gvZmV0Y2hcIik7XG5cbi8qKiBUaGUgc2xpY2Ugb2YgYSBnb3Qtc2NyYXBpbmcgcmVzcG9uc2UgdGhpcyBtb2R1bGUgcmVhZHMuICovXG5pbnRlcmZhY2UgRmV0Y2hlZFBhZ2Uge1xuICB1cmw6IHN0cmluZztcbiAgc3RhdHVzQ29kZTogbnVtYmVyO1xuICBjb250ZW50VHlwZTogc3RyaW5nIHwgdW5kZWZpbmVkO1xuICBib2R5OiBzdHJpbmc7XG59XG5cbi8qKiBQdWxsIHRoZSA8dGl0bGU+IHRleHQgb3V0IG9mIGFuIEhUTUwgZG9jdW1lbnQsIHdoZW4gcHJlc2VudC4gKi9cbmZ1bmN0aW9uIGV4dHJhY3RUaXRsZShodG1sOiBzdHJpbmcpOiBzdHJpbmcgfCB1bmRlZmluZWQge1xuICBjb25zdCBtYXRjaCA9IC88dGl0bGVbXj5dKj4oW1xcc1xcU10qPyk8XFwvdGl0bGU+L2kuZXhlYyhodG1sKTtcbiAgY29uc3QgdGl0bGUgPSBtYXRjaD8uWzFdID8gZGVjb2RlSHRtbEVudGl0aWVzKG1hdGNoWzFdKS50cmltKCkgOiBcIlwiO1xuICByZXR1cm4gdGl0bGUubGVuZ3RoID4gMCA/IHRpdGxlIDogdW5kZWZpbmVkO1xufVxuXG4vKiogVHJ1bmNhdGUgYHRleHRgIHRvIGBtYXhgIGNoYXJzLCByZXBvcnRpbmcgd2hldGhlciBpdCB3YXMgY3V0LiAqL1xuZnVuY3Rpb24gdHJ1bmNhdGUodGV4dDogc3RyaW5nLCBtYXg6IG51bWJlcik6IHsgY29udGVudDogc3RyaW5nOyB0cnVuY2F0ZWQ6IGJvb2xlYW4gfSB7XG4gIGlmICh0ZXh0Lmxlbmd0aCA8PSBtYXgpIHJldHVybiB7IGNvbnRlbnQ6IHRleHQsIHRydW5jYXRlZDogZmFsc2UgfTtcbiAgcmV0dXJuIHsgY29udGVudDogdGV4dC5zbGljZSgwLCBtYXgpLCB0cnVuY2F0ZWQ6IHRydWUgfTtcbn1cblxuLyoqXG4gKiBGZXRjaCBhIHNpbmdsZSBVUkwgYW5kIHJldHVybiBpdHMgY29udGVudCBpbiB0aGUgcmVxdWVzdGVkIGZvcm1hdC5cbiAqXG4gKiBUaHJvd3Mgd2hlbiB0aGUgVVJMIGlzIG5vdCBwZXJtaXR0ZWQgYnkgdGhlIGFsbG93LWxpc3QgKHRoZSB2aXNpYmxlLFxuICogY29ycmVjdGFibGUgZmFpbHVyZSB0aGUgZGVzaWduIGNhbGxzIGZvciBvbiB0aGUgZmV0Y2ggcGF0aCkuIFRoZSByZXF1ZXN0J3NcbiAqIGBtYXhMZW5ndGhgIG5hcnJvd3MgKG5ldmVyIHdpZGVucykgdGhlIHBsdWdpbidzIGBmZXRjaE1heExlbmd0aGAgY2FwLlxuICogYHNpZ25hbGAgY2FuY2VscyB0aGUgaW4tZmxpZ2h0IGZldGNoLlxuICovXG5leHBvcnQgYXN5bmMgZnVuY3Rpb24gcnVuV2ViRmV0Y2goXG4gIHJlcXVlc3Q6IFdlYkZldGNoUmVxdWVzdCxcbiAgcnVudGltZU9yQ29uZmlnOiBXZWJTZWFyY2hSdW50aW1lSW5wdXQsXG4gIHNpZ25hbD86IEFib3J0U2lnbmFsLFxuKTogUHJvbWlzZTxXZWJGZXRjaFJlc3VsdD4ge1xuICBjb25zdCBydW50aW1lID0gdG9XZWJTZWFyY2hSdW50aW1lKHJ1bnRpbWVPckNvbmZpZyk7XG4gIGNvbnN0IHsgY29uZmlnIH0gPSBydW50aW1lO1xuICBhd2FpdCBhc3NlcnRGZXRjaFVybEFsbG93ZWQocmVxdWVzdC51cmwsIGNvbmZpZy5hbGxvd0xpc3QpO1xuICBjb25zdCBjYXAgPSBNYXRoLm1pbihyZXF1ZXN0Lm1heExlbmd0aCA/PyBjb25maWcuZmV0Y2hNYXhMZW5ndGgsIGNvbmZpZy5mZXRjaE1heExlbmd0aCk7XG5cbiAgLy8gVGhlIHJlc3BvbnNlIGlzIGNhY2hlZCBiZWZvcmUgdGhlIGZvcm1hdCByZWR1Y3Rpb24sIHNvIHRoZSBzYW1lIHBhZ2UgcmVhZFxuICAvLyBhcyB0ZXh0IGFuZCBhcyBodG1sIHNoYXJlcyBvbmUgbmV0d29yayByb3VuZCB0cmlwLlxuICBjb25zdCBwYWdlID0gYXdhaXQgZXhlY3V0ZVJlYWQoXG4gICAgcnVudGltZSxcbiAgICBcInBhZ2UtZmV0Y2hcIixcbiAgICB0b0NhbGxTZXR0aW5ncyh3ZWJGZXRjaEV4ZWN1dGVEZWZhdWx0cywgY29uZmlnLnRpbWVvdXRNcywgW1xuICAgICAgXCJ3ZWItc2VhcmNoXCIsXG4gICAgICBcImZldGNoXCIsXG4gICAgICByZXF1ZXN0LnVybCxcbiAgICAgIC4uLmNvbmZpZy5hbGxvd0xpc3QucGF0dGVybnMsXG4gICAgXSksXG4gICAgYXN5bmMgKGV4ZWN1dGVTaWduYWwpOiBQcm9taXNlPEZldGNoZWRQYWdlPiA9PiB7XG4gICAgICBjb25zdCByZXNwb25zZSA9IGF3YWl0IGdvdFNjcmFwaW5nKHtcbiAgICAgICAgdXJsOiByZXF1ZXN0LnVybCxcbiAgICAgICAgLy8gQnVuJ3Mgbm9kZTpodHRwMiByZXBvcnRzIGRlZmF1bHQtcG9ydCBvcmlnaW5zIHdpdGggOjQ0MyB3aGlsZVxuICAgICAgICAvLyBodHRwMi13cmFwcGVyIGNvbXBhcmVzIHRoZW0gYWdhaW5zdCBVUkwtbm9ybWFsaXplZCBvcmlnaW5zLlxuICAgICAgICBodHRwMjogZmFsc2UsXG4gICAgICAgIC8vIGdvdCdzIG93biB0aW1lb3V0IGFib3J0cyB0aGUgc29ja2V0IGFuZCByZXBvcnRzIHdoaWNoIHBoYXNlIHRpbWVkXG4gICAgICAgIC8vIG91dDsgdGhlIGludGVyY2VwdG9yIHRpbWVvdXQgYm91bmRzIHRoZSB3aG9sZSBhdHRlbXB0IGFyb3VuZCBpdC5cbiAgICAgICAgdGltZW91dDogeyByZXF1ZXN0OiBjb25maWcudGltZW91dE1zIH0sXG4gICAgICAgIHRocm93SHR0cEVycm9yczogZmFsc2UsXG4gICAgICAgIGZvbGxvd1JlZGlyZWN0OiB0cnVlLFxuICAgICAgICBob29rczoge1xuICAgICAgICAgIGJlZm9yZVJlcXVlc3Q6IFtcbiAgICAgICAgICAgIGFzeW5jIChvcHRpb25zKSA9PiB7XG4gICAgICAgICAgICAgIGF3YWl0IGFzc2VydEZldGNoVXJsQWxsb3dlZChTdHJpbmcob3B0aW9ucy51cmwpLCBjb25maWcuYWxsb3dMaXN0KTtcbiAgICAgICAgICAgIH0sXG4gICAgICAgICAgXSxcbiAgICAgICAgfSxcbiAgICAgICAgLi4uKGV4ZWN1dGVTaWduYWwgPyB7IHNpZ25hbDogZXhlY3V0ZVNpZ25hbCB9IDoge30pLFxuICAgICAgfSk7XG4gICAgICByZXR1cm4ge1xuICAgICAgICB1cmw6IHJlc3BvbnNlLnVybCA/PyByZXF1ZXN0LnVybCxcbiAgICAgICAgc3RhdHVzQ29kZTogcmVzcG9uc2Uuc3RhdHVzQ29kZSxcbiAgICAgICAgY29udGVudFR5cGU6IHJlc3BvbnNlLmhlYWRlcnNbXCJjb250ZW50LXR5cGVcIl0sXG4gICAgICAgIGJvZHk6IHR5cGVvZiByZXNwb25zZS5ib2R5ID09PSBcInN0cmluZ1wiID8gcmVzcG9uc2UuYm9keSA6IFN0cmluZyhyZXNwb25zZS5ib2R5ID8/IFwiXCIpLFxuICAgICAgfTtcbiAgICB9LFxuICAgIHNpZ25hbCxcbiAgKTtcbiAgYXdhaXQgYXNzZXJ0RmV0Y2hVcmxBbGxvd2VkKHBhZ2UudXJsLCBjb25maWcuYWxsb3dMaXN0KTtcblxuICBjb25zdCBpc0h0bWwgPSAhcGFnZS5jb250ZW50VHlwZSB8fCAvaHRtbHx4bWwvaS50ZXN0KHBhZ2UuY29udGVudFR5cGUpO1xuICBjb25zdCByYXdDb250ZW50ID0gcmVxdWVzdC5mb3JtYXQgPT09IFwiaHRtbFwiIHx8ICFpc0h0bWwgPyBwYWdlLmJvZHkgOiBodG1sVG9UZXh0KHBhZ2UuYm9keSk7XG4gIGNvbnN0IHsgY29udGVudCwgdHJ1bmNhdGVkIH0gPSB0cnVuY2F0ZShyYXdDb250ZW50LCBjYXApO1xuICBjb25zdCB0aXRsZSA9IGlzSHRtbCA/IGV4dHJhY3RUaXRsZShwYWdlLmJvZHkpIDogdW5kZWZpbmVkO1xuXG4gIGxvZ2dlci5kZWJ1ZyhcImZldGNoZWRcIiwge1xuICAgIHVybDogcGFnZS51cmwsXG4gICAgc3RhdHVzOiBwYWdlLnN0YXR1c0NvZGUsXG4gICAgYnl0ZXM6IHBhZ2UuYm9keS5sZW5ndGgsXG4gICAgcmV0dXJuZWQ6IGNvbnRlbnQubGVuZ3RoLFxuICAgIC4uLih0cnVuY2F0ZWQgPyB7IHRydW5jYXRlZDogdHJ1ZSB9IDoge30pLFxuICB9KTtcblxuICByZXR1cm4ge1xuICAgIHVybDogcGFnZS51cmwsXG4gICAgc3RhdHVzOiBwYWdlLnN0YXR1c0NvZGUsXG4gICAgLi4uKHBhZ2UuY29udGVudFR5cGUgPyB7IGNvbnRlbnRUeXBlOiBwYWdlLmNvbnRlbnRUeXBlIH0gOiB7fSksXG4gICAgLi4uKHRpdGxlID8geyB0aXRsZSB9IDoge30pLFxuICAgIGNvbnRlbnQsXG4gICAgdHJ1bmNhdGVkLFxuICB9O1xufVxuIl19
|
|
110
|
+
/**
|
|
111
|
+
* Fetch bounded raw markup for deterministic in-process asset discovery.
|
|
112
|
+
*
|
|
113
|
+
* This does not widen the model-facing `web_fetch` cap.
|
|
114
|
+
*/
|
|
115
|
+
export async function runWebAssetPageFetch(request, runtimeOrConfig, signal) {
|
|
116
|
+
const runtime = toWebSearchRuntime(runtimeOrConfig);
|
|
117
|
+
const cap = Math.min(Math.max(1, request.maxLength ?? DEFAULT_ASSET_PAGE_MAX_LENGTH), DEFAULT_ASSET_PAGE_MAX_LENGTH);
|
|
118
|
+
const page = await fetchPage(request.url, runtime, signal);
|
|
119
|
+
const { content, truncated } = truncate(page.body, cap);
|
|
120
|
+
const title = extractTitle(page.body);
|
|
121
|
+
return {
|
|
122
|
+
url: page.url,
|
|
123
|
+
status: page.statusCode,
|
|
124
|
+
...(page.contentType ? { contentType: page.contentType } : {}),
|
|
125
|
+
...(title ? { title } : {}),
|
|
126
|
+
content,
|
|
127
|
+
truncated,
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
/** Fetch one public image while preserving URL policy, timeout, and redirect checks. */
|
|
131
|
+
export async function runWebImageFetch(request, runtimeOrConfig, signal) {
|
|
132
|
+
const runtime = toWebSearchRuntime(runtimeOrConfig);
|
|
133
|
+
const { config } = runtime;
|
|
134
|
+
const maxBytes = Math.max(1, request.maxBytes ?? DEFAULT_IMAGE_MAX_BYTES);
|
|
135
|
+
await assertFetchUrlAllowed(request.url, config.allowList);
|
|
136
|
+
const page = await executeRead(runtime, "image-fetch", {
|
|
137
|
+
default: {
|
|
138
|
+
cache: { enabled: false },
|
|
139
|
+
retry: { enabled: true, attempts: 2 },
|
|
140
|
+
timeout: config.timeoutMs,
|
|
141
|
+
},
|
|
142
|
+
}, async (executeSignal) => {
|
|
143
|
+
const response = await gotScraping({
|
|
144
|
+
url: request.url,
|
|
145
|
+
responseType: "buffer",
|
|
146
|
+
http2: false,
|
|
147
|
+
timeout: { request: config.timeoutMs },
|
|
148
|
+
throwHttpErrors: true,
|
|
149
|
+
followRedirect: true,
|
|
150
|
+
hooks: {
|
|
151
|
+
beforeRequest: [
|
|
152
|
+
async (options) => {
|
|
153
|
+
await assertFetchUrlAllowed(String(options.url), config.allowList);
|
|
154
|
+
},
|
|
155
|
+
],
|
|
156
|
+
},
|
|
157
|
+
...(executeSignal ? { signal: executeSignal } : {}),
|
|
158
|
+
});
|
|
159
|
+
const contentType = String(response.headers["content-type"] ?? "")
|
|
160
|
+
.split(";")[0]
|
|
161
|
+
.trim()
|
|
162
|
+
.toLowerCase();
|
|
163
|
+
if (!contentType.startsWith("image/")) {
|
|
164
|
+
throw new TypeError(`Expected an image response, received ${contentType || "unknown"}`);
|
|
165
|
+
}
|
|
166
|
+
if (response.body.byteLength > maxBytes) {
|
|
167
|
+
throw new RangeError(`Image response exceeds ${maxBytes} bytes`);
|
|
168
|
+
}
|
|
169
|
+
return {
|
|
170
|
+
url: response.url ?? request.url,
|
|
171
|
+
contentType,
|
|
172
|
+
data: new Uint8Array(response.body),
|
|
173
|
+
};
|
|
174
|
+
}, signal, false);
|
|
175
|
+
await assertFetchUrlAllowed(page.url, config.allowList);
|
|
176
|
+
return page;
|
|
177
|
+
}
|
|
178
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmV0Y2guanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvZmV0Y2gudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUE7Ozs7Ozs7Ozs7Ozs7OztHQWVHO0FBRUgsT0FBTyxFQUFFLEdBQUcsRUFBRSxNQUFNLHdCQUF3QixDQUFDO0FBQzdDLE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSxjQUFjLENBQUM7QUFDM0MsT0FBTyxFQUFFLHFCQUFxQixFQUFFLE1BQU0saUJBQWlCLENBQUM7QUFDeEQsT0FBTyxFQUFFLGNBQWMsRUFBRSx1QkFBdUIsRUFBRSxNQUFNLGVBQWUsQ0FBQztBQUN4RSxPQUFPLEVBQUUsa0JBQWtCLEVBQUUsVUFBVSxFQUFFLE1BQU0sZ0JBQWdCLENBQUM7QUFDaEUsT0FBTyxFQUNMLFdBQVcsRUFDWCxrQkFBa0IsR0FHbkIsTUFBTSxjQUFjLENBQUM7QUFHdEIsTUFBTSxNQUFNLEdBQUcsR0FBRyxDQUFDLE1BQU0sQ0FBQyxrQkFBa0IsQ0FBQyxDQUFDO0FBQzlDLE1BQU0sdUJBQXVCLEdBQUcsQ0FBQyxHQUFHLElBQUksR0FBRyxJQUFJLENBQUM7QUFDaEQsTUFBTSw2QkFBNkIsR0FBRyxTQUFTLENBQUM7QUFvQ2hELG1FQUFtRTtBQUNuRSxTQUFTLFlBQVksQ0FBQyxJQUFZO0lBQ2hDLE1BQU0sS0FBSyxHQUFHLGtDQUFrQyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQztJQUM1RCxNQUFNLEtBQUssR0FBRyxLQUFLLEVBQUUsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsa0JBQWtCLENBQUMsS0FBSyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsSUFBSSxFQUFFLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQztJQUNwRSxPQUFPLEtBQUssQ0FBQyxNQUFNLEdBQUcsQ0FBQyxDQUFDLENBQUMsQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDLFNBQVMsQ0FBQztBQUM5QyxDQUFDO0FBRUQsb0VBQW9FO0FBQ3BFLFNBQVMsUUFBUSxDQUFDLElBQVksRUFBRSxHQUFXO0lBQ3pDLElBQUksSUFBSSxDQUFDLE1BQU0sSUFBSSxHQUFHO1FBQUUsT0FBTyxFQUFFLE9BQU8sRUFBRSxJQUFJLEVBQUUsU0FBUyxFQUFFLEtBQUssRUFBRSxDQUFDO0lBQ25FLE9BQU8sRUFBRSxPQUFPLEVBQUUsSUFBSSxDQUFDLEtBQUssQ0FBQyxDQUFDLEVBQUUsR0FBRyxDQUFDLEVBQUUsU0FBUyxFQUFFLElBQUksRUFBRSxDQUFDO0FBQzFELENBQUM7QUFFRCw0RUFBNEU7QUFDNUUsS0FBSyxVQUFVLFNBQVMsQ0FDdEIsVUFBa0IsRUFDbEIsT0FBeUIsRUFDekIsTUFBb0I7SUFFcEIsTUFBTSxFQUFFLE1BQU0sRUFBRSxHQUFHLE9BQU8sQ0FBQztJQUMzQixNQUFNLHFCQUFxQixDQUFDLFVBQVUsRUFBRSxNQUFNLENBQUMsU0FBUyxDQUFDLENBQUM7SUFDMUQsTUFBTSxJQUFJLEdBQUcsTUFBTSxXQUFXLENBQzVCLE9BQU8sRUFDUCxZQUFZLEVBQ1osY0FBYyxDQUFDLHVCQUF1QixFQUFFLE1BQU0sQ0FBQyxTQUFTLEVBQUU7UUFDeEQsWUFBWTtRQUNaLE9BQU87UUFDUCxVQUFVO1FBQ1YsR0FBRyxNQUFNLENBQUMsU0FBUyxDQUFDLFFBQVE7S0FDN0IsQ0FBQyxFQUNGLEtBQUssRUFBRSxhQUFhLEVBQXdCLEVBQUU7UUFDNUMsTUFBTSxRQUFRLEdBQUcsTUFBTSxXQUFXLENBQUM7WUFDakMsR0FBRyxFQUFFLFVBQVU7WUFDZixnRUFBZ0U7WUFDaEUsOERBQThEO1lBQzlELEtBQUssRUFBRSxLQUFLO1lBQ1osb0VBQW9FO1lBQ3BFLG1FQUFtRTtZQUNuRSxPQUFPLEVBQUUsRUFBRSxPQUFPLEVBQUUsTUFBTSxDQUFDLFNBQVMsRUFBRTtZQUN0QyxlQUFlLEVBQUUsS0FBSztZQUN0QixjQUFjLEVBQUUsSUFBSTtZQUNwQixLQUFLLEVBQUU7Z0JBQ0wsYUFBYSxFQUFFO29CQUNiLEtBQUssRUFBRSxPQUFPLEVBQUUsRUFBRTt3QkFDaEIsTUFBTSxxQkFBcUIsQ0FBQyxNQUFNLENBQUMsT0FBTyxDQUFDLEdBQUcsQ0FBQyxFQUFFLE1BQU0sQ0FBQyxTQUFTLENBQUMsQ0FBQztvQkFDckUsQ0FBQztpQkFDRjthQUNGO1lBQ0QsR0FBRyxDQUFDLGFBQWEsQ0FBQyxDQUFDLENBQUMsRUFBRSxNQUFNLEVBQUUsYUFBYSxFQUFFLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQztTQUNwRCxDQUFDLENBQUM7UUFDSCxPQUFPO1lBQ0wsR0FBRyxFQUFFLFFBQVEsQ0FBQyxHQUFHLElBQUksVUFBVTtZQUMvQixVQUFVLEVBQUUsUUFBUSxDQUFDLFVBQVU7WUFDL0IsV0FBVyxFQUFFLFFBQVEsQ0FBQyxPQUFPLENBQUMsY0FBYyxDQUFDO1lBQzdDLElBQUksRUFBRSxPQUFPLFFBQVEsQ0FBQyxJQUFJLEtBQUssUUFBUSxDQUFDLENBQUMsQ0FBQyxRQUFRLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxNQUFNLENBQUMsUUFBUSxDQUFDLElBQUksSUFBSSxFQUFFLENBQUM7U0FDdEYsQ0FBQztJQUNKLENBQUMsRUFDRCxNQUFNLENBQ1AsQ0FBQztJQUNGLE1BQU0scUJBQXFCLENBQUMsSUFBSSxDQUFDLEdBQUcsRUFBRSxNQUFNLENBQUMsU0FBUyxDQUFDLENBQUM7SUFDeEQsT0FBTyxJQUFJLENBQUM7QUFDZCxDQUFDO0FBRUQ7Ozs7Ozs7R0FPRztBQUNILE1BQU0sQ0FBQyxLQUFLLFVBQVUsV0FBVyxDQUMvQixPQUF3QixFQUN4QixlQUFzQyxFQUN0QyxNQUFvQjtJQUVwQixNQUFNLE9BQU8sR0FBRyxrQkFBa0IsQ0FBQyxlQUFlLENBQUMsQ0FBQztJQUNwRCxNQUFNLEVBQUUsTUFBTSxFQUFFLEdBQUcsT0FBTyxDQUFDO0lBQzNCLE1BQU0sR0FBRyxHQUFHLElBQUksQ0FBQyxHQUFHLENBQUMsT0FBTyxDQUFDLFNBQVMsSUFBSSxNQUFNLENBQUMsY0FBYyxFQUFFLE1BQU0sQ0FBQyxjQUFjLENBQUMsQ0FBQztJQUN4RixNQUFNLElBQUksR0FBRyxNQUFNLFNBQVMsQ0FBQyxPQUFPLENBQUMsR0FBRyxFQUFFLE9BQU8sRUFBRSxNQUFNLENBQUMsQ0FBQztJQUUzRCxNQUFNLE1BQU0sR0FBRyxDQUFDLElBQUksQ0FBQyxXQUFXLElBQUksV0FBVyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsV0FBVyxDQUFDLENBQUM7SUFDdkUsTUFBTSxVQUFVLEdBQUcsT0FBTyxDQUFDLE1BQU0sS0FBSyxNQUFNLElBQUksQ0FBQyxNQUFNLENBQUMsQ0FBQyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLFVBQVUsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUM7SUFDNUYsTUFBTSxFQUFFLE9BQU8sRUFBRSxTQUFTLEVBQUUsR0FBRyxRQUFRLENBQUMsVUFBVSxFQUFFLEdBQUcsQ0FBQyxDQUFDO0lBQ3pELE1BQU0sS0FBSyxHQUFHLE1BQU0sQ0FBQyxDQUFDLENBQUMsWUFBWSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLENBQUMsU0FBUyxDQUFDO0lBRTNELE1BQU0sQ0FBQyxLQUFLLENBQUMsU0FBUyxFQUFFO1FBQ3RCLEdBQUcsRUFBRSxJQUFJLENBQUMsR0FBRztRQUNiLE1BQU0sRUFBRSxJQUFJLENBQUMsVUFBVTtRQUN2QixLQUFLLEVBQUUsSUFBSSxDQUFDLElBQUksQ0FBQyxNQUFNO1FBQ3ZCLFFBQVEsRUFBRSxPQUFPLENBQUMsTUFBTTtRQUN4QixHQUFHLENBQUMsU0FBUyxDQUFDLENBQUMsQ0FBQyxFQUFFLFNBQVMsRUFBRSxJQUFJLEVBQUUsQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDO0tBQzFDLENBQUMsQ0FBQztJQUVILE9BQU87UUFDTCxHQUFHLEVBQUUsSUFBSSxDQUFDLEdBQUc7UUFDYixNQUFNLEVBQUUsSUFBSSxDQUFDLFVBQVU7UUFDdkIsR0FBRyxDQUFDLElBQUksQ0FBQyxXQUFXLENBQUMsQ0FBQyxDQUFDLEVBQUUsV0FBVyxFQUFFLElBQUksQ0FBQyxXQUFXLEVBQUUsQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDO1FBQzlELEdBQUcsQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDLEVBQUUsS0FBSyxFQUFFLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQztRQUMzQixPQUFPO1FBQ1AsU0FBUztLQUNWLENBQUM7QUFDSixDQUFDO0FBRUQ7Ozs7R0FJRztBQUNILE1BQU0sQ0FBQyxLQUFLLFVBQVUsb0JBQW9CLENBQ3hDLE9BQWlDLEVBQ2pDLGVBQXNDLEVBQ3RDLE1BQW9CO0lBRXBCLE1BQU0sT0FBTyxHQUFHLGtCQUFrQixDQUFDLGVBQWUsQ0FBQyxDQUFDO0lBQ3BELE1BQU0sR0FBRyxHQUFHLElBQUksQ0FBQyxHQUFHLENBQ2xCLElBQUksQ0FBQyxHQUFHLENBQUMsQ0FBQyxFQUFFLE9BQU8sQ0FBQyxTQUFTLElBQUksNkJBQTZCLENBQUMsRUFDL0QsNkJBQTZCLENBQzlCLENBQUM7SUFDRixNQUFNLElBQUksR0FBRyxNQUFNLFNBQVMsQ0FBQyxPQUFPLENBQUMsR0FBRyxFQUFFLE9BQU8sRUFBRSxNQUFNLENBQUMsQ0FBQztJQUMzRCxNQUFNLEVBQUUsT0FBTyxFQUFFLFNBQVMsRUFBRSxHQUFHLFFBQVEsQ0FBQyxJQUFJLENBQUMsSUFBSSxFQUFFLEdBQUcsQ0FBQyxDQUFDO0lBQ3hELE1BQU0sS0FBSyxHQUFHLFlBQVksQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUM7SUFDdEMsT0FBTztRQUNMLEdBQUcsRUFBRSxJQUFJLENBQUMsR0FBRztRQUNiLE1BQU0sRUFBRSxJQUFJLENBQUMsVUFBVTtRQUN2QixHQUFHLENBQUMsSUFBSSxDQUFDLFdBQVcsQ0FBQyxDQUFDLENBQUMsRUFBRSxXQUFXLEVBQUUsSUFBSSxDQUFDLFdBQVcsRUFBRSxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUM7UUFDOUQsR0FBRyxDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUMsRUFBRSxLQUFLLEVBQUUsQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDO1FBQzNCLE9BQU87UUFDUCxTQUFTO0tBQ1YsQ0FBQztBQUNKLENBQUM7QUFFRCx3RkFBd0Y7QUFDeEYsTUFBTSxDQUFDLEtBQUssVUFBVSxnQkFBZ0IsQ0FDcEMsT0FBNkIsRUFDN0IsZUFBc0MsRUFDdEMsTUFBb0I7SUFFcEIsTUFBTSxPQUFPLEdBQUcsa0JBQWtCLENBQUMsZUFBZSxDQUFDLENBQUM7SUFDcEQsTUFBTSxFQUFFLE1BQU0sRUFBRSxHQUFHLE9BQU8sQ0FBQztJQUMzQixNQUFNLFFBQVEsR0FBRyxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUMsRUFBRSxPQUFPLENBQUMsUUFBUSxJQUFJLHVCQUF1QixDQUFDLENBQUM7SUFDMUUsTUFBTSxxQkFBcUIsQ0FBQyxPQUFPLENBQUMsR0FBRyxFQUFFLE1BQU0sQ0FBQyxTQUFTLENBQUMsQ0FBQztJQUMzRCxNQUFNLElBQUksR0FBRyxNQUFNLFdBQVcsQ0FDNUIsT0FBTyxFQUNQLGFBQWEsRUFDYjtRQUNFLE9BQU8sRUFBRTtZQUNQLEtBQUssRUFBRSxFQUFFLE9BQU8sRUFBRSxLQUFLLEVBQUU7WUFDekIsS0FBSyxFQUFFLEVBQUUsT0FBTyxFQUFFLElBQUksRUFBRSxRQUFRLEVBQUUsQ0FBQyxFQUFFO1lBQ3JDLE9BQU8sRUFBRSxNQUFNLENBQUMsU0FBUztTQUMxQjtLQUNGLEVBQ0QsS0FBSyxFQUFFLGFBQWEsRUFBRSxFQUFFO1FBQ3RCLE1BQU0sUUFBUSxHQUFHLE1BQU0sV0FBVyxDQUFDO1lBQ2pDLEdBQUcsRUFBRSxPQUFPLENBQUMsR0FBRztZQUNoQixZQUFZLEVBQUUsUUFBUTtZQUN0QixLQUFLLEVBQUUsS0FBSztZQUNaLE9BQU8sRUFBRSxFQUFFLE9BQU8sRUFBRSxNQUFNLENBQUMsU0FBUyxFQUFFO1lBQ3RDLGVBQWUsRUFBRSxJQUFJO1lBQ3JCLGNBQWMsRUFBRSxJQUFJO1lBQ3BCLEtBQUssRUFBRTtnQkFDTCxhQUFhLEVBQUU7b0JBQ2IsS0FBSyxFQUFFLE9BQU8sRUFBRSxFQUFFO3dCQUNoQixNQUFNLHFCQUFxQixDQUFDLE1BQU0sQ0FBQyxPQUFPLENBQUMsR0FBRyxDQUFDLEVBQUUsTUFBTSxDQUFDLFNBQVMsQ0FBQyxDQUFDO29CQUNyRSxDQUFDO2lCQUNGO2FBQ0Y7WUFDRCxHQUFHLENBQUMsYUFBYSxDQUFDLENBQUMsQ0FBQyxFQUFFLE1BQU0sRUFBRSxhQUFhLEVBQUUsQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDO1NBQ3BELENBQUMsQ0FBQztRQUNILE1BQU0sV0FBVyxHQUFHLE1BQU0sQ0FBQyxRQUFRLENBQUMsT0FBTyxDQUFDLGNBQWMsQ0FBQyxJQUFJLEVBQUUsQ0FBQzthQUMvRCxLQUFLLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQyxDQUFFO2FBQ2QsSUFBSSxFQUFFO2FBQ04sV0FBVyxFQUFFLENBQUM7UUFDakIsSUFBSSxDQUFDLFdBQVcsQ0FBQyxVQUFVLENBQUMsUUFBUSxDQUFDLEVBQUUsQ0FBQztZQUN0QyxNQUFNLElBQUksU0FBUyxDQUFDLHdDQUF3QyxXQUFXLElBQUksU0FBUyxFQUFFLENBQUMsQ0FBQztRQUMxRixDQUFDO1FBQ0QsSUFBSSxRQUFRLENBQUMsSUFBSSxDQUFDLFVBQVUsR0FBRyxRQUFRLEVBQUUsQ0FBQztZQUN4QyxNQUFNLElBQUksVUFBVSxDQUFDLDBCQUEwQixRQUFRLFFBQVEsQ0FBQyxDQUFDO1FBQ25FLENBQUM7UUFDRCxPQUFPO1lBQ0wsR0FBRyxFQUFFLFFBQVEsQ0FBQyxHQUFHLElBQUksT0FBTyxDQUFDLEdBQUc7WUFDaEMsV0FBVztZQUNYLElBQUksRUFBRSxJQUFJLFVBQVUsQ0FBQyxRQUFRLENBQUMsSUFBSSxDQUFDO1NBQ3BDLENBQUM7SUFDSixDQUFDLEVBQ0QsTUFBTSxFQUNOLEtBQUssQ0FDTixDQUFDO0lBQ0YsTUFBTSxxQkFBcUIsQ0FBQyxJQUFJLENBQUMsR0FBRyxFQUFFLE1BQU0sQ0FBQyxTQUFTLENBQUMsQ0FBQztJQUN4RCxPQUFPLElBQUksQ0FBQztBQUNkLENBQUMiLCJzb3VyY2VzQ29udGVudCI6WyIvKipcbiAqIFBhZ2UgZmV0Y2hpbmcgb3ZlciB7QGxpbmsgaHR0cHM6Ly93d3cubnBtanMuY29tL3BhY2thZ2UvZ290LXNjcmFwaW5nIHwgZ290LXNjcmFwaW5nfVxuICogLSBhIGBnb3RgIHdyYXBwZXIgdGhhdCBnZW5lcmF0ZXMgYnJvd3Nlci1saWtlIFRMUyArIGhlYWRlciBmaW5nZXJwcmludHMsIHNvXG4gKiBmZXRjaGVzIHN1cnZpdmUgdGhlIGJvdCB3YWxscyBhIHBsYWluIGBmZXRjaGAgdHJpcHMuIHtAbGluayBydW5XZWJGZXRjaH1cbiAqIGVuZm9yY2VzIHRoZSBVUkwgYWxsb3ctbGlzdCAoYW4gZXhwbGljaXQgZmV0Y2ggb2YgYSBkaXNhbGxvd2VkIFVSTCBpc1xuICogcmVmdXNlZCwgbm90IHNpbGVudGx5IGVtcHRpZWQpLCBmZXRjaGVzIHdpdGggdGhlIHBsdWdpbidzIHRpbWVvdXQsIGFuZFxuICogcmV0dXJucyBlaXRoZXIgdGhlIHJhdyBIVE1MIG9yIGEgcmVhZGFibGUgcGxhaW4tdGV4dCByZWR1Y3Rpb24sIGNhcHBlZCBhdFxuICogdGhlIGNvbmZpZ3VyZWQgbGVuZ3RoLlxuICpcbiAqIFRoZSBIVE1MLXRvLXRleHQgcmVkdWN0aW9uIGlzIGRlbGliZXJhdGVseSBkZXBlbmRlbmN5LWZyZWUgKHN0cmlwXG4gKiBzY3JpcHQvc3R5bGUsIHVud3JhcCB0YWdzLCBkZWNvZGUgYSBoYW5kZnVsIG9mIGVudGl0aWVzLCBjb2xsYXBzZVxuICogd2hpdGVzcGFjZSk6IGdvb2QgZW5vdWdoIHRvIGZlZWQgYSBtb2RlbCwgYW5kIGl0IGtlZXBzIHRoZSBhZGQtb24nc1xuICogZGVwZW5kZW5jeSBzdXJmYWNlIHRvIHRoZSB0d28gbGlicmFyaWVzIHRoZSB0YXNrIGNhbGxlZCBmb3IuXG4gKlxuICogQG1vZHVsZVxuICovXG5cbmltcG9ydCB7IGxvZyB9IGZyb20gXCJAZGJ4LXRvb2xzL3NoYXJlZC1jb3JlXCI7XG5pbXBvcnQgeyBnb3RTY3JhcGluZyB9IGZyb20gXCJnb3Qtc2NyYXBpbmdcIjtcbmltcG9ydCB7IGFzc2VydEZldGNoVXJsQWxsb3dlZCB9IGZyb20gXCIuL19mZXRjaC11cmwudHNcIjtcbmltcG9ydCB7IHRvQ2FsbFNldHRpbmdzLCB3ZWJGZXRjaEV4ZWN1dGVEZWZhdWx0cyB9IGZyb20gXCIuL2RlZmF1bHRzLnRzXCI7XG5pbXBvcnQgeyBkZWNvZGVIdG1sRW50aXRpZXMsIGh0bWxUb1RleHQgfSBmcm9tIFwiLi9odG1sLXRleHQudHNcIjtcbmltcG9ydCB7XG4gIGV4ZWN1dGVSZWFkLFxuICB0b1dlYlNlYXJjaFJ1bnRpbWUsXG4gIHR5cGUgV2ViU2VhcmNoUnVudGltZSxcbiAgdHlwZSBXZWJTZWFyY2hSdW50aW1lSW5wdXQsXG59IGZyb20gXCIuL3J1bnRpbWUudHNcIjtcbmltcG9ydCB0eXBlIHsgV2ViRmV0Y2hSZXF1ZXN0LCBXZWJGZXRjaFJlc3VsdCB9IGZyb20gXCIuL3NjaGVtYS50c1wiO1xuXG5jb25zdCBsb2dnZXIgPSBsb2cubG9nZ2VyKFwid2ViLXNlYXJjaC9mZXRjaFwiKTtcbmNvbnN0IERFRkFVTFRfSU1BR0VfTUFYX0JZVEVTID0gNSAqIDEwMjQgKiAxMDI0O1xuY29uc3QgREVGQVVMVF9BU1NFVF9QQUdFX01BWF9MRU5HVEggPSAxXzUwMF8wMDA7XG5cbi8qKiBJbnB1dCBmb3IgYSBwb2xpY3ktY2hlY2tlZCBIVE1MIGZldGNoIHVzZWQgYnkgaW4tcHJvY2VzcyBhc3NldCBkaXNjb3ZlcnkuICovXG5leHBvcnQgaW50ZXJmYWNlIFdlYkFzc2V0UGFnZUZldGNoUmVxdWVzdCB7XG4gIC8qKiBQdWJsaWMgSFRUUFMgcGFnZSBVUkwuICovXG4gIHVybDogc3RyaW5nO1xuICAvKiogTWF4aW11bSByZXR1cm5lZCBtYXJrdXAgbGVuZ3RoLiBEZWZhdWx0cyB0byAxLjUgbWlsbGlvbiBjaGFyYWN0ZXJzLiAqL1xuICBtYXhMZW5ndGg/OiBudW1iZXI7XG59XG5cbi8qKiBJbnB1dCBmb3IgYSBwb2xpY3ktY2hlY2tlZCBiaW5hcnkgaW1hZ2UgZmV0Y2guICovXG5leHBvcnQgaW50ZXJmYWNlIFdlYkltYWdlRmV0Y2hSZXF1ZXN0IHtcbiAgLyoqIFB1YmxpYyBIVFRQUyBpbWFnZSBVUkwuICovXG4gIHVybDogc3RyaW5nO1xuICAvKiogTWF4aW11bSBhY2NlcHRlZCByZXNwb25zZSBzaXplLiBEZWZhdWx0cyB0byA1IE1pQi4gKi9cbiAgbWF4Qnl0ZXM/OiBudW1iZXI7XG59XG5cbi8qKiBCaW5hcnkgaW1hZ2UgZmV0Y2hlZCB0aHJvdWdoIHRoZSB3ZWItc2VhcmNoIHJ1bnRpbWUncyBVUkwgcG9saWN5LiAqL1xuZXhwb3J0IGludGVyZmFjZSBXZWJJbWFnZUZldGNoUmVzdWx0IHtcbiAgLyoqIEZpbmFsIFVSTCBhZnRlciByZWRpcmVjdHMuICovXG4gIHVybDogc3RyaW5nO1xuICAvKiogSW1hZ2UgbWVkaWEgdHlwZSByZXR1cm5lZCBieSB0aGUgc2VydmVyLiAqL1xuICBjb250ZW50VHlwZTogc3RyaW5nO1xuICAvKiogRmV0Y2hlZCBpbWFnZSBieXRlcy4gKi9cbiAgZGF0YTogVWludDhBcnJheTtcbn1cblxuLyoqIFRoZSBzbGljZSBvZiBhIGdvdC1zY3JhcGluZyByZXNwb25zZSB0aGlzIG1vZHVsZSByZWFkcy4gKi9cbmludGVyZmFjZSBGZXRjaGVkUGFnZSB7XG4gIHVybDogc3RyaW5nO1xuICBzdGF0dXNDb2RlOiBudW1iZXI7XG4gIGNvbnRlbnRUeXBlOiBzdHJpbmcgfCB1bmRlZmluZWQ7XG4gIGJvZHk6IHN0cmluZztcbn1cblxuLyoqIFB1bGwgdGhlIDx0aXRsZT4gdGV4dCBvdXQgb2YgYW4gSFRNTCBkb2N1bWVudCwgd2hlbiBwcmVzZW50LiAqL1xuZnVuY3Rpb24gZXh0cmFjdFRpdGxlKGh0bWw6IHN0cmluZyk6IHN0cmluZyB8IHVuZGVmaW5lZCB7XG4gIGNvbnN0IG1hdGNoID0gLzx0aXRsZVtePl0qPihbXFxzXFxTXSo/KTxcXC90aXRsZT4vaS5leGVjKGh0bWwpO1xuICBjb25zdCB0aXRsZSA9IG1hdGNoPy5bMV0gPyBkZWNvZGVIdG1sRW50aXRpZXMobWF0Y2hbMV0pLnRyaW0oKSA6IFwiXCI7XG4gIHJldHVybiB0aXRsZS5sZW5ndGggPiAwID8gdGl0bGUgOiB1bmRlZmluZWQ7XG59XG5cbi8qKiBUcnVuY2F0ZSBgdGV4dGAgdG8gYG1heGAgY2hhcnMsIHJlcG9ydGluZyB3aGV0aGVyIGl0IHdhcyBjdXQuICovXG5mdW5jdGlvbiB0cnVuY2F0ZSh0ZXh0OiBzdHJpbmcsIG1heDogbnVtYmVyKTogeyBjb250ZW50OiBzdHJpbmc7IHRydW5jYXRlZDogYm9vbGVhbiB9IHtcbiAgaWYgKHRleHQubGVuZ3RoIDw9IG1heCkgcmV0dXJuIHsgY29udGVudDogdGV4dCwgdHJ1bmNhdGVkOiBmYWxzZSB9O1xuICByZXR1cm4geyBjb250ZW50OiB0ZXh0LnNsaWNlKDAsIG1heCksIHRydW5jYXRlZDogdHJ1ZSB9O1xufVxuXG4vKiogRmV0Y2ggYW5kIGNhY2hlIG9uZSByYXcgcGFnZSBiZWZvcmUgY2FsbGVyLXNwZWNpZmljIG91dHB1dCByZWR1Y3Rpb24uICovXG5hc3luYyBmdW5jdGlvbiBmZXRjaFBhZ2UoXG4gIHJlcXVlc3RVcmw6IHN0cmluZyxcbiAgcnVudGltZTogV2ViU2VhcmNoUnVudGltZSxcbiAgc2lnbmFsPzogQWJvcnRTaWduYWwsXG4pOiBQcm9taXNlPEZldGNoZWRQYWdlPiB7XG4gIGNvbnN0IHsgY29uZmlnIH0gPSBydW50aW1lO1xuICBhd2FpdCBhc3NlcnRGZXRjaFVybEFsbG93ZWQocmVxdWVzdFVybCwgY29uZmlnLmFsbG93TGlzdCk7XG4gIGNvbnN0IHBhZ2UgPSBhd2FpdCBleGVjdXRlUmVhZChcbiAgICBydW50aW1lLFxuICAgIFwicGFnZS1mZXRjaFwiLFxuICAgIHRvQ2FsbFNldHRpbmdzKHdlYkZldGNoRXhlY3V0ZURlZmF1bHRzLCBjb25maWcudGltZW91dE1zLCBbXG4gICAgICBcIndlYi1zZWFyY2hcIixcbiAgICAgIFwiZmV0Y2hcIixcbiAgICAgIHJlcXVlc3RVcmwsXG4gICAgICAuLi5jb25maWcuYWxsb3dMaXN0LnBhdHRlcm5zLFxuICAgIF0pLFxuICAgIGFzeW5jIChleGVjdXRlU2lnbmFsKTogUHJvbWlzZTxGZXRjaGVkUGFnZT4gPT4ge1xuICAgICAgY29uc3QgcmVzcG9uc2UgPSBhd2FpdCBnb3RTY3JhcGluZyh7XG4gICAgICAgIHVybDogcmVxdWVzdFVybCxcbiAgICAgICAgLy8gQnVuJ3Mgbm9kZTpodHRwMiByZXBvcnRzIGRlZmF1bHQtcG9ydCBvcmlnaW5zIHdpdGggOjQ0MyB3aGlsZVxuICAgICAgICAvLyBodHRwMi13cmFwcGVyIGNvbXBhcmVzIHRoZW0gYWdhaW5zdCBVUkwtbm9ybWFsaXplZCBvcmlnaW5zLlxuICAgICAgICBodHRwMjogZmFsc2UsXG4gICAgICAgIC8vIGdvdCdzIG93biB0aW1lb3V0IGFib3J0cyB0aGUgc29ja2V0IGFuZCByZXBvcnRzIHdoaWNoIHBoYXNlIHRpbWVkXG4gICAgICAgIC8vIG91dDsgdGhlIGludGVyY2VwdG9yIHRpbWVvdXQgYm91bmRzIHRoZSB3aG9sZSBhdHRlbXB0IGFyb3VuZCBpdC5cbiAgICAgICAgdGltZW91dDogeyByZXF1ZXN0OiBjb25maWcudGltZW91dE1zIH0sXG4gICAgICAgIHRocm93SHR0cEVycm9yczogZmFsc2UsXG4gICAgICAgIGZvbGxvd1JlZGlyZWN0OiB0cnVlLFxuICAgICAgICBob29rczoge1xuICAgICAgICAgIGJlZm9yZVJlcXVlc3Q6IFtcbiAgICAgICAgICAgIGFzeW5jIChvcHRpb25zKSA9PiB7XG4gICAgICAgICAgICAgIGF3YWl0IGFzc2VydEZldGNoVXJsQWxsb3dlZChTdHJpbmcob3B0aW9ucy51cmwpLCBjb25maWcuYWxsb3dMaXN0KTtcbiAgICAgICAgICAgIH0sXG4gICAgICAgICAgXSxcbiAgICAgICAgfSxcbiAgICAgICAgLi4uKGV4ZWN1dGVTaWduYWwgPyB7IHNpZ25hbDogZXhlY3V0ZVNpZ25hbCB9IDoge30pLFxuICAgICAgfSk7XG4gICAgICByZXR1cm4ge1xuICAgICAgICB1cmw6IHJlc3BvbnNlLnVybCA/PyByZXF1ZXN0VXJsLFxuICAgICAgICBzdGF0dXNDb2RlOiByZXNwb25zZS5zdGF0dXNDb2RlLFxuICAgICAgICBjb250ZW50VHlwZTogcmVzcG9uc2UuaGVhZGVyc1tcImNvbnRlbnQtdHlwZVwiXSxcbiAgICAgICAgYm9keTogdHlwZW9mIHJlc3BvbnNlLmJvZHkgPT09IFwic3RyaW5nXCIgPyByZXNwb25zZS5ib2R5IDogU3RyaW5nKHJlc3BvbnNlLmJvZHkgPz8gXCJcIiksXG4gICAgICB9O1xuICAgIH0sXG4gICAgc2lnbmFsLFxuICApO1xuICBhd2FpdCBhc3NlcnRGZXRjaFVybEFsbG93ZWQocGFnZS51cmwsIGNvbmZpZy5hbGxvd0xpc3QpO1xuICByZXR1cm4gcGFnZTtcbn1cblxuLyoqXG4gKiBGZXRjaCBhIHNpbmdsZSBVUkwgYW5kIHJldHVybiBpdHMgY29udGVudCBpbiB0aGUgcmVxdWVzdGVkIGZvcm1hdC5cbiAqXG4gKiBUaHJvd3Mgd2hlbiB0aGUgVVJMIGlzIG5vdCBwZXJtaXR0ZWQgYnkgdGhlIGFsbG93LWxpc3QgKHRoZSB2aXNpYmxlLFxuICogY29ycmVjdGFibGUgZmFpbHVyZSB0aGUgZGVzaWduIGNhbGxzIGZvciBvbiB0aGUgZmV0Y2ggcGF0aCkuIFRoZSByZXF1ZXN0J3NcbiAqIGBtYXhMZW5ndGhgIG5hcnJvd3MgKG5ldmVyIHdpZGVucykgdGhlIHBsdWdpbidzIGBmZXRjaE1heExlbmd0aGAgY2FwLlxuICogYHNpZ25hbGAgY2FuY2VscyB0aGUgaW4tZmxpZ2h0IGZldGNoLlxuICovXG5leHBvcnQgYXN5bmMgZnVuY3Rpb24gcnVuV2ViRmV0Y2goXG4gIHJlcXVlc3Q6IFdlYkZldGNoUmVxdWVzdCxcbiAgcnVudGltZU9yQ29uZmlnOiBXZWJTZWFyY2hSdW50aW1lSW5wdXQsXG4gIHNpZ25hbD86IEFib3J0U2lnbmFsLFxuKTogUHJvbWlzZTxXZWJGZXRjaFJlc3VsdD4ge1xuICBjb25zdCBydW50aW1lID0gdG9XZWJTZWFyY2hSdW50aW1lKHJ1bnRpbWVPckNvbmZpZyk7XG4gIGNvbnN0IHsgY29uZmlnIH0gPSBydW50aW1lO1xuICBjb25zdCBjYXAgPSBNYXRoLm1pbihyZXF1ZXN0Lm1heExlbmd0aCA/PyBjb25maWcuZmV0Y2hNYXhMZW5ndGgsIGNvbmZpZy5mZXRjaE1heExlbmd0aCk7XG4gIGNvbnN0IHBhZ2UgPSBhd2FpdCBmZXRjaFBhZ2UocmVxdWVzdC51cmwsIHJ1bnRpbWUsIHNpZ25hbCk7XG5cbiAgY29uc3QgaXNIdG1sID0gIXBhZ2UuY29udGVudFR5cGUgfHwgL2h0bWx8eG1sL2kudGVzdChwYWdlLmNvbnRlbnRUeXBlKTtcbiAgY29uc3QgcmF3Q29udGVudCA9IHJlcXVlc3QuZm9ybWF0ID09PSBcImh0bWxcIiB8fCAhaXNIdG1sID8gcGFnZS5ib2R5IDogaHRtbFRvVGV4dChwYWdlLmJvZHkpO1xuICBjb25zdCB7IGNvbnRlbnQsIHRydW5jYXRlZCB9ID0gdHJ1bmNhdGUocmF3Q29udGVudCwgY2FwKTtcbiAgY29uc3QgdGl0bGUgPSBpc0h0bWwgPyBleHRyYWN0VGl0bGUocGFnZS5ib2R5KSA6IHVuZGVmaW5lZDtcblxuICBsb2dnZXIuZGVidWcoXCJmZXRjaGVkXCIsIHtcbiAgICB1cmw6IHBhZ2UudXJsLFxuICAgIHN0YXR1czogcGFnZS5zdGF0dXNDb2RlLFxuICAgIGJ5dGVzOiBwYWdlLmJvZHkubGVuZ3RoLFxuICAgIHJldHVybmVkOiBjb250ZW50Lmxlbmd0aCxcbiAgICAuLi4odHJ1bmNhdGVkID8geyB0cnVuY2F0ZWQ6IHRydWUgfSA6IHt9KSxcbiAgfSk7XG5cbiAgcmV0dXJuIHtcbiAgICB1cmw6IHBhZ2UudXJsLFxuICAgIHN0YXR1czogcGFnZS5zdGF0dXNDb2RlLFxuICAgIC4uLihwYWdlLmNvbnRlbnRUeXBlID8geyBjb250ZW50VHlwZTogcGFnZS5jb250ZW50VHlwZSB9IDoge30pLFxuICAgIC4uLih0aXRsZSA/IHsgdGl0bGUgfSA6IHt9KSxcbiAgICBjb250ZW50LFxuICAgIHRydW5jYXRlZCxcbiAgfTtcbn1cblxuLyoqXG4gKiBGZXRjaCBib3VuZGVkIHJhdyBtYXJrdXAgZm9yIGRldGVybWluaXN0aWMgaW4tcHJvY2VzcyBhc3NldCBkaXNjb3ZlcnkuXG4gKlxuICogVGhpcyBkb2VzIG5vdCB3aWRlbiB0aGUgbW9kZWwtZmFjaW5nIGB3ZWJfZmV0Y2hgIGNhcC5cbiAqL1xuZXhwb3J0IGFzeW5jIGZ1bmN0aW9uIHJ1bldlYkFzc2V0UGFnZUZldGNoKFxuICByZXF1ZXN0OiBXZWJBc3NldFBhZ2VGZXRjaFJlcXVlc3QsXG4gIHJ1bnRpbWVPckNvbmZpZzogV2ViU2VhcmNoUnVudGltZUlucHV0LFxuICBzaWduYWw/OiBBYm9ydFNpZ25hbCxcbik6IFByb21pc2U8V2ViRmV0Y2hSZXN1bHQ+IHtcbiAgY29uc3QgcnVudGltZSA9IHRvV2ViU2VhcmNoUnVudGltZShydW50aW1lT3JDb25maWcpO1xuICBjb25zdCBjYXAgPSBNYXRoLm1pbihcbiAgICBNYXRoLm1heCgxLCByZXF1ZXN0Lm1heExlbmd0aCA/PyBERUZBVUxUX0FTU0VUX1BBR0VfTUFYX0xFTkdUSCksXG4gICAgREVGQVVMVF9BU1NFVF9QQUdFX01BWF9MRU5HVEgsXG4gICk7XG4gIGNvbnN0IHBhZ2UgPSBhd2FpdCBmZXRjaFBhZ2UocmVxdWVzdC51cmwsIHJ1bnRpbWUsIHNpZ25hbCk7XG4gIGNvbnN0IHsgY29udGVudCwgdHJ1bmNhdGVkIH0gPSB0cnVuY2F0ZShwYWdlLmJvZHksIGNhcCk7XG4gIGNvbnN0IHRpdGxlID0gZXh0cmFjdFRpdGxlKHBhZ2UuYm9keSk7XG4gIHJldHVybiB7XG4gICAgdXJsOiBwYWdlLnVybCxcbiAgICBzdGF0dXM6IHBhZ2Uuc3RhdHVzQ29kZSxcbiAgICAuLi4ocGFnZS5jb250ZW50VHlwZSA/IHsgY29udGVudFR5cGU6IHBhZ2UuY29udGVudFR5cGUgfSA6IHt9KSxcbiAgICAuLi4odGl0bGUgPyB7IHRpdGxlIH0gOiB7fSksXG4gICAgY29udGVudCxcbiAgICB0cnVuY2F0ZWQsXG4gIH07XG59XG5cbi8qKiBGZXRjaCBvbmUgcHVibGljIGltYWdlIHdoaWxlIHByZXNlcnZpbmcgVVJMIHBvbGljeSwgdGltZW91dCwgYW5kIHJlZGlyZWN0IGNoZWNrcy4gKi9cbmV4cG9ydCBhc3luYyBmdW5jdGlvbiBydW5XZWJJbWFnZUZldGNoKFxuICByZXF1ZXN0OiBXZWJJbWFnZUZldGNoUmVxdWVzdCxcbiAgcnVudGltZU9yQ29uZmlnOiBXZWJTZWFyY2hSdW50aW1lSW5wdXQsXG4gIHNpZ25hbD86IEFib3J0U2lnbmFsLFxuKTogUHJvbWlzZTxXZWJJbWFnZUZldGNoUmVzdWx0PiB7XG4gIGNvbnN0IHJ1bnRpbWUgPSB0b1dlYlNlYXJjaFJ1bnRpbWUocnVudGltZU9yQ29uZmlnKTtcbiAgY29uc3QgeyBjb25maWcgfSA9IHJ1bnRpbWU7XG4gIGNvbnN0IG1heEJ5dGVzID0gTWF0aC5tYXgoMSwgcmVxdWVzdC5tYXhCeXRlcyA/PyBERUZBVUxUX0lNQUdFX01BWF9CWVRFUyk7XG4gIGF3YWl0IGFzc2VydEZldGNoVXJsQWxsb3dlZChyZXF1ZXN0LnVybCwgY29uZmlnLmFsbG93TGlzdCk7XG4gIGNvbnN0IHBhZ2UgPSBhd2FpdCBleGVjdXRlUmVhZChcbiAgICBydW50aW1lLFxuICAgIFwiaW1hZ2UtZmV0Y2hcIixcbiAgICB7XG4gICAgICBkZWZhdWx0OiB7XG4gICAgICAgIGNhY2hlOiB7IGVuYWJsZWQ6IGZhbHNlIH0sXG4gICAgICAgIHJldHJ5OiB7IGVuYWJsZWQ6IHRydWUsIGF0dGVtcHRzOiAyIH0sXG4gICAgICAgIHRpbWVvdXQ6IGNvbmZpZy50aW1lb3V0TXMsXG4gICAgICB9LFxuICAgIH0sXG4gICAgYXN5bmMgKGV4ZWN1dGVTaWduYWwpID0+IHtcbiAgICAgIGNvbnN0IHJlc3BvbnNlID0gYXdhaXQgZ290U2NyYXBpbmcoe1xuICAgICAgICB1cmw6IHJlcXVlc3QudXJsLFxuICAgICAgICByZXNwb25zZVR5cGU6IFwiYnVmZmVyXCIsXG4gICAgICAgIGh0dHAyOiBmYWxzZSxcbiAgICAgICAgdGltZW91dDogeyByZXF1ZXN0OiBjb25maWcudGltZW91dE1zIH0sXG4gICAgICAgIHRocm93SHR0cEVycm9yczogdHJ1ZSxcbiAgICAgICAgZm9sbG93UmVkaXJlY3Q6IHRydWUsXG4gICAgICAgIGhvb2tzOiB7XG4gICAgICAgICAgYmVmb3JlUmVxdWVzdDogW1xuICAgICAgICAgICAgYXN5bmMgKG9wdGlvbnMpID0+IHtcbiAgICAgICAgICAgICAgYXdhaXQgYXNzZXJ0RmV0Y2hVcmxBbGxvd2VkKFN0cmluZyhvcHRpb25zLnVybCksIGNvbmZpZy5hbGxvd0xpc3QpO1xuICAgICAgICAgICAgfSxcbiAgICAgICAgICBdLFxuICAgICAgICB9LFxuICAgICAgICAuLi4oZXhlY3V0ZVNpZ25hbCA/IHsgc2lnbmFsOiBleGVjdXRlU2lnbmFsIH0gOiB7fSksXG4gICAgICB9KTtcbiAgICAgIGNvbnN0IGNvbnRlbnRUeXBlID0gU3RyaW5nKHJlc3BvbnNlLmhlYWRlcnNbXCJjb250ZW50LXR5cGVcIl0gPz8gXCJcIilcbiAgICAgICAgLnNwbGl0KFwiO1wiKVswXSFcbiAgICAgICAgLnRyaW0oKVxuICAgICAgICAudG9Mb3dlckNhc2UoKTtcbiAgICAgIGlmICghY29udGVudFR5cGUuc3RhcnRzV2l0aChcImltYWdlL1wiKSkge1xuICAgICAgICB0aHJvdyBuZXcgVHlwZUVycm9yKGBFeHBlY3RlZCBhbiBpbWFnZSByZXNwb25zZSwgcmVjZWl2ZWQgJHtjb250ZW50VHlwZSB8fCBcInVua25vd25cIn1gKTtcbiAgICAgIH1cbiAgICAgIGlmIChyZXNwb25zZS5ib2R5LmJ5dGVMZW5ndGggPiBtYXhCeXRlcykge1xuICAgICAgICB0aHJvdyBuZXcgUmFuZ2VFcnJvcihgSW1hZ2UgcmVzcG9uc2UgZXhjZWVkcyAke21heEJ5dGVzfSBieXRlc2ApO1xuICAgICAgfVxuICAgICAgcmV0dXJuIHtcbiAgICAgICAgdXJsOiByZXNwb25zZS51cmwgPz8gcmVxdWVzdC51cmwsXG4gICAgICAgIGNvbnRlbnRUeXBlLFxuICAgICAgICBkYXRhOiBuZXcgVWludDhBcnJheShyZXNwb25zZS5ib2R5KSxcbiAgICAgIH07XG4gICAgfSxcbiAgICBzaWduYWwsXG4gICAgZmFsc2UsXG4gICk7XG4gIGF3YWl0IGFzc2VydEZldGNoVXJsQWxsb3dlZChwYWdlLnVybCwgY29uZmlnLmFsbG93TGlzdCk7XG4gIHJldHVybiBwYWdlO1xufVxuIl19
|
package/lib/src/plugin.d.ts
CHANGED
|
@@ -19,6 +19,7 @@ import { type ToolRegistry } from "@databricks/appkit/beta";
|
|
|
19
19
|
import { ToolRegistryPlugin } from "@dbx-tools/appkit";
|
|
20
20
|
import type { AgentToolExecutionContext } from "@dbx-tools/appkit/tool-provider";
|
|
21
21
|
import { type WebSearchPluginConfig } from "./config.ts";
|
|
22
|
+
import { type WebAssetPageFetchRequest, type WebImageFetchRequest, type WebImageFetchResult } from "./fetch.ts";
|
|
22
23
|
import type { WebFetchRequest, WebFetchResult, WebSearchRequest, WebSearchResult } from "./schema.ts";
|
|
23
24
|
/**
|
|
24
25
|
* AppKit plugin that resolves and holds the web-search runtime config used
|
|
@@ -123,6 +124,10 @@ export declare class WebSearchPlugin extends ToolRegistryPlugin<WebSearchPluginC
|
|
|
123
124
|
* configured URL policy.
|
|
124
125
|
*/
|
|
125
126
|
fetch: (request: WebFetchRequest, signal?: AbortSignal) => Promise<WebFetchResult>;
|
|
127
|
+
/** Fetch expanded HTML for in-process asset discovery. */
|
|
128
|
+
fetchAssetPage: (request: WebAssetPageFetchRequest, signal?: AbortSignal) => Promise<WebFetchResult>;
|
|
129
|
+
/** Fetch one image for in-process visual research. */
|
|
130
|
+
fetchImage: (request: WebImageFetchRequest, signal?: AbortSignal) => Promise<WebImageFetchResult>;
|
|
126
131
|
};
|
|
127
132
|
private search;
|
|
128
133
|
private fetch;
|