@expandai/mcp-server 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +190 -0
- package/README.md +67 -0
- package/dist/main.cjs +814 -0
- package/package.json +61 -0
- package/src/ExpandClient.ts +26 -0
- package/src/generated/ExpandClient.ts +821 -0
- package/src/main.ts +24 -0
- package/src/resources/ExpandDocs.ts +14 -0
- package/src/tools/Fetch.ts +94 -0
- package/src/tools/index.ts +33 -0
|
@@ -0,0 +1,821 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @since 1.0.0
|
|
3
|
+
*/
|
|
4
|
+
import type * as HttpClient from '@effect/platform/HttpClient'
|
|
5
|
+
import * as HttpClientError from '@effect/platform/HttpClientError'
|
|
6
|
+
import * as HttpClientRequest from '@effect/platform/HttpClientRequest'
|
|
7
|
+
import * as HttpClientResponse from '@effect/platform/HttpClientResponse'
|
|
8
|
+
import * as Data from 'effect/Data'
|
|
9
|
+
import * as Effect from 'effect/Effect'
|
|
10
|
+
import type { ParseError } from 'effect/ParseResult'
|
|
11
|
+
import * as S from 'effect/Schema'
|
|
12
|
+
|
|
13
|
+
export class AssetsGetByUrlParams extends S.Struct({
|
|
14
|
+
url: S.String,
|
|
15
|
+
}) {}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* The tag identifying the type of parse issue
|
|
19
|
+
*/
|
|
20
|
+
export class IssueTag extends S.Literal(
|
|
21
|
+
'Pointer',
|
|
22
|
+
'Unexpected',
|
|
23
|
+
'Missing',
|
|
24
|
+
'Composite',
|
|
25
|
+
'Refinement',
|
|
26
|
+
'Transformation',
|
|
27
|
+
'Type',
|
|
28
|
+
'Forbidden',
|
|
29
|
+
) {}
|
|
30
|
+
|
|
31
|
+
export class PropertyKeyEnumTag extends S.Literal('symbol') {}
|
|
32
|
+
|
|
33
|
+
export class PropertyKey extends S.Union(
|
|
34
|
+
S.String,
|
|
35
|
+
S.Number,
|
|
36
|
+
/**
|
|
37
|
+
* an object to be decoded into a globally shared symbol
|
|
38
|
+
*/
|
|
39
|
+
S.Struct({
|
|
40
|
+
_tag: PropertyKeyEnumTag,
|
|
41
|
+
key: S.String,
|
|
42
|
+
}),
|
|
43
|
+
) {}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Represents an error encountered while parsing a value to match the schema
|
|
47
|
+
*/
|
|
48
|
+
export class Issue extends S.Class<Issue>('Issue')({
|
|
49
|
+
/**
|
|
50
|
+
* The tag identifying the type of parse issue
|
|
51
|
+
*/
|
|
52
|
+
_tag: IssueTag,
|
|
53
|
+
/**
|
|
54
|
+
* The path to the property where the issue occurred
|
|
55
|
+
*/
|
|
56
|
+
path: S.Array(PropertyKey),
|
|
57
|
+
/**
|
|
58
|
+
* A descriptive message explaining the issue
|
|
59
|
+
*/
|
|
60
|
+
message: S.String,
|
|
61
|
+
}) {}
|
|
62
|
+
|
|
63
|
+
export class HttpApiDecodeErrorTag extends S.Literal('HttpApiDecodeError') {}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* The request did not match the expected schema
|
|
67
|
+
*/
|
|
68
|
+
export class HttpApiDecodeError extends S.Class<HttpApiDecodeError>('HttpApiDecodeError')({
|
|
69
|
+
issues: S.Array(Issue),
|
|
70
|
+
message: S.String,
|
|
71
|
+
_tag: HttpApiDecodeErrorTag,
|
|
72
|
+
}) {}
|
|
73
|
+
|
|
74
|
+
export class UnauthorizedAccessTag extends S.Literal('UnauthorizedAccess') {}
|
|
75
|
+
|
|
76
|
+
export class UnauthorizedAccess extends S.Class<UnauthorizedAccess>('UnauthorizedAccess')({
|
|
77
|
+
_tag: UnauthorizedAccessTag,
|
|
78
|
+
}) {}
|
|
79
|
+
|
|
80
|
+
export class AssetNotFoundTag extends S.Literal('AssetNotFound') {}
|
|
81
|
+
|
|
82
|
+
export class AssetNotFound extends S.Class<AssetNotFound>('AssetNotFound')({
|
|
83
|
+
browserSessionId: S.optionalWith(S.String, { nullable: true }),
|
|
84
|
+
filename: S.String,
|
|
85
|
+
_tag: AssetNotFoundTag,
|
|
86
|
+
}) {}
|
|
87
|
+
|
|
88
|
+
export class AssetNotInWaczTag extends S.Literal('AssetNotInWacz') {}
|
|
89
|
+
|
|
90
|
+
export class AssetNotInWacz extends S.Class<AssetNotInWacz>('AssetNotInWacz')({
|
|
91
|
+
url: S.String,
|
|
92
|
+
_tag: AssetNotInWaczTag,
|
|
93
|
+
}) {}
|
|
94
|
+
|
|
95
|
+
export class AssetsGetByUrl404 extends S.Union(AssetNotFound, AssetNotInWacz) {}
|
|
96
|
+
|
|
97
|
+
export class AssetReadErrorTag extends S.Literal('AssetReadError') {}
|
|
98
|
+
|
|
99
|
+
export class AssetReadError extends S.Class<AssetReadError>('AssetReadError')({
|
|
100
|
+
browserSessionId: S.optionalWith(S.String, { nullable: true }),
|
|
101
|
+
filename: S.String,
|
|
102
|
+
_tag: AssetReadErrorTag,
|
|
103
|
+
}) {}
|
|
104
|
+
|
|
105
|
+
export class InvalidWaczFormatTag extends S.Literal('InvalidWaczFormat') {}
|
|
106
|
+
|
|
107
|
+
export class InvalidWaczFormat extends S.Class<InvalidWaczFormat>('InvalidWaczFormat')({
|
|
108
|
+
message: S.String,
|
|
109
|
+
_tag: InvalidWaczFormatTag,
|
|
110
|
+
}) {}
|
|
111
|
+
|
|
112
|
+
export class AssetRetrievalErrorTag extends S.Literal('AssetRetrievalError') {}
|
|
113
|
+
|
|
114
|
+
export class AssetRetrievalError extends S.Class<AssetRetrievalError>('AssetRetrievalError')({
|
|
115
|
+
browserSessionId: S.String,
|
|
116
|
+
url: S.String,
|
|
117
|
+
_tag: AssetRetrievalErrorTag,
|
|
118
|
+
}) {}
|
|
119
|
+
|
|
120
|
+
export class AssetsGetByUrl500 extends S.Union(AssetReadError, InvalidWaczFormat, AssetRetrievalError) {}
|
|
121
|
+
|
|
122
|
+
export class CurlErrorInputMethod extends S.Literal(
|
|
123
|
+
'GET',
|
|
124
|
+
'POST',
|
|
125
|
+
'HEAD',
|
|
126
|
+
'PUT',
|
|
127
|
+
'DELETE',
|
|
128
|
+
'CONNECT',
|
|
129
|
+
'OPTIONS',
|
|
130
|
+
'TRACE',
|
|
131
|
+
'PATCH',
|
|
132
|
+
) {}
|
|
133
|
+
|
|
134
|
+
export class CurlErrorTag extends S.Literal('CurlError') {}
|
|
135
|
+
|
|
136
|
+
export class CurlError extends S.Class<CurlError>('CurlError')({
|
|
137
|
+
status: S.Number,
|
|
138
|
+
input: S.Struct({
|
|
139
|
+
method: CurlErrorInputMethod,
|
|
140
|
+
url: S.String,
|
|
141
|
+
body: S.optionalWith(S.String, { nullable: true }),
|
|
142
|
+
headers: S.optionalWith(S.Struct({}), { nullable: true }),
|
|
143
|
+
proxy: S.optionalWith(
|
|
144
|
+
S.Struct({
|
|
145
|
+
server: S.String,
|
|
146
|
+
username: S.optionalWith(S.String, { nullable: true }),
|
|
147
|
+
password: S.optionalWith(S.String, { nullable: true }),
|
|
148
|
+
}),
|
|
149
|
+
{ nullable: true },
|
|
150
|
+
),
|
|
151
|
+
timeout: S.optionalWith(S.Number, { nullable: true }),
|
|
152
|
+
connectTimeout: S.optionalWith(S.Number, { nullable: true }),
|
|
153
|
+
}),
|
|
154
|
+
_tag: CurlErrorTag,
|
|
155
|
+
}) {}
|
|
156
|
+
|
|
157
|
+
export class AssetListItem extends S.Class<AssetListItem>('AssetListItem')({
|
|
158
|
+
url: S.String,
|
|
159
|
+
mime: S.String,
|
|
160
|
+
status: S.Number,
|
|
161
|
+
}) {}
|
|
162
|
+
|
|
163
|
+
export class AssetListResponse extends S.Class<AssetListResponse>('AssetListResponse')({
|
|
164
|
+
browserSessionId: S.String,
|
|
165
|
+
assets: S.Array(AssetListItem),
|
|
166
|
+
}) {}
|
|
167
|
+
|
|
168
|
+
export class AssetsList500 extends S.Union(AssetReadError, InvalidWaczFormat) {}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* a string to be decoded into a URL
|
|
172
|
+
*/
|
|
173
|
+
export class URL extends S.String {}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Options for customizing HTML content extraction
|
|
177
|
+
*/
|
|
178
|
+
export class SelectHtmlConfig extends S.Record({ key: S.String, value: S.Unknown }) {}
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* Options for customizing markdown content extraction
|
|
182
|
+
*/
|
|
183
|
+
export class SelectMarkdownConfig extends S.Record({ key: S.String, value: S.Unknown }) {}
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* Options for customizing screenshot capture behavior
|
|
187
|
+
*/
|
|
188
|
+
export class SelectScreenshotConfig extends S.Class<SelectScreenshotConfig>('SelectScreenshotConfig')({
|
|
189
|
+
/**
|
|
190
|
+
* Whether to capture the full page including content below the fold
|
|
191
|
+
*/
|
|
192
|
+
fullPage: S.optionalWith(S.Boolean, { nullable: true }),
|
|
193
|
+
}) {}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* a string with no leading or trailing whitespace
|
|
197
|
+
*/
|
|
198
|
+
export class Trimmed extends S.String.pipe(S.pattern(/^\S[\s\S]*\S$|^\S$|^$/)) {}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* an integer
|
|
202
|
+
*/
|
|
203
|
+
export class Int extends S.Int {}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Request parameters for fetching and extracting content from a web page
|
|
207
|
+
*/
|
|
208
|
+
export class FetchRequest extends S.Class<FetchRequest>('FetchRequest')({
|
|
209
|
+
url: URL,
|
|
210
|
+
/**
|
|
211
|
+
* Specifies which content formats to include in the response
|
|
212
|
+
*/
|
|
213
|
+
select: S.optionalWith(
|
|
214
|
+
S.Struct({
|
|
215
|
+
html: S.optionalWith(
|
|
216
|
+
S.Union(
|
|
217
|
+
/**
|
|
218
|
+
* Set to true to include HTML
|
|
219
|
+
*/
|
|
220
|
+
S.Boolean,
|
|
221
|
+
SelectHtmlConfig,
|
|
222
|
+
),
|
|
223
|
+
{ nullable: true },
|
|
224
|
+
),
|
|
225
|
+
/**
|
|
226
|
+
* Include markdown-formatted content in the response
|
|
227
|
+
*/
|
|
228
|
+
markdown: S.optionalWith(
|
|
229
|
+
S.Union(
|
|
230
|
+
/**
|
|
231
|
+
* Set to true to include markdown
|
|
232
|
+
*/
|
|
233
|
+
S.Boolean,
|
|
234
|
+
SelectMarkdownConfig,
|
|
235
|
+
),
|
|
236
|
+
{ nullable: true },
|
|
237
|
+
),
|
|
238
|
+
screenshot: S.optionalWith(
|
|
239
|
+
S.Union(
|
|
240
|
+
/**
|
|
241
|
+
* Set to true to include a screenshot
|
|
242
|
+
*/
|
|
243
|
+
S.Boolean,
|
|
244
|
+
SelectScreenshotConfig,
|
|
245
|
+
),
|
|
246
|
+
{ nullable: true },
|
|
247
|
+
),
|
|
248
|
+
summary: S.optionalWith(
|
|
249
|
+
S.Union(
|
|
250
|
+
/**
|
|
251
|
+
* Set to true to generate an AI summary
|
|
252
|
+
*/
|
|
253
|
+
S.Boolean,
|
|
254
|
+
/**
|
|
255
|
+
* Options for AI-powered page summarization
|
|
256
|
+
*/
|
|
257
|
+
S.Struct({
|
|
258
|
+
/**
|
|
259
|
+
* Custom prompt for AI summarization (max 5000 characters)
|
|
260
|
+
*/
|
|
261
|
+
prompt: S.optionalWith(S.String.pipe(S.maxLength(5000)), {
|
|
262
|
+
nullable: true,
|
|
263
|
+
default: () =>
|
|
264
|
+
'You are a helpful assistant that summarizes web page content.\nGiven the markdown content of a web page, provide a clear and concise summary.\nFocus on the main points and key information.\nKeep the summary informative but brief.' as const,
|
|
265
|
+
}),
|
|
266
|
+
}),
|
|
267
|
+
),
|
|
268
|
+
{ nullable: true },
|
|
269
|
+
),
|
|
270
|
+
/**
|
|
271
|
+
* Options for extracting relevant snippets from page content using semantic search
|
|
272
|
+
*/
|
|
273
|
+
snippets: S.optionalWith(
|
|
274
|
+
S.Struct({
|
|
275
|
+
/**
|
|
276
|
+
* Query to find relevant content snippets from the page (required, non-empty)
|
|
277
|
+
*/
|
|
278
|
+
query: Trimmed,
|
|
279
|
+
/**
|
|
280
|
+
* Maximum number of snippets to return (1-50)
|
|
281
|
+
*/
|
|
282
|
+
maxSnippets: S.optionalWith(Int, { nullable: true, default: () => 5 as const }),
|
|
283
|
+
/**
|
|
284
|
+
* Minimum relevance score threshold (0-1). Snippets below this score are filtered out.
|
|
285
|
+
*/
|
|
286
|
+
minScore: S.optionalWith(S.Number.pipe(S.greaterThanOrEqualTo(0), S.lessThanOrEqualTo(1)), {
|
|
287
|
+
nullable: true,
|
|
288
|
+
default: () => 0.5 as const,
|
|
289
|
+
}),
|
|
290
|
+
/**
|
|
291
|
+
* Target snippet size in characters (100-2000)
|
|
292
|
+
*/
|
|
293
|
+
targetSnippetSize: S.optionalWith(Int, { nullable: true, default: () => 384 as const }),
|
|
294
|
+
}),
|
|
295
|
+
{ nullable: true },
|
|
296
|
+
),
|
|
297
|
+
/**
|
|
298
|
+
* Include page metadata in the response
|
|
299
|
+
*/
|
|
300
|
+
meta: S.optionalWith(S.Boolean, { nullable: true, default: () => true as const }),
|
|
301
|
+
/**
|
|
302
|
+
* Configure response info options (headers inclusion)
|
|
303
|
+
*/
|
|
304
|
+
response: S.optionalWith(
|
|
305
|
+
S.Struct({
|
|
306
|
+
/**
|
|
307
|
+
* Whether to include HTTP response headers
|
|
308
|
+
*/
|
|
309
|
+
includeHeaders: S.optionalWith(S.Boolean, { nullable: true, default: () => false as const }),
|
|
310
|
+
}),
|
|
311
|
+
{ nullable: true },
|
|
312
|
+
),
|
|
313
|
+
}),
|
|
314
|
+
{ nullable: true },
|
|
315
|
+
),
|
|
316
|
+
/**
|
|
317
|
+
* Configuration options for browser behavior during the fetch
|
|
318
|
+
*/
|
|
319
|
+
browserConfig: S.optionalWith(
|
|
320
|
+
S.Struct({
|
|
321
|
+
/**
|
|
322
|
+
* Whether to scroll the entire page to capture lazy-loaded content
|
|
323
|
+
*/
|
|
324
|
+
scrollFullPage: S.optionalWith(S.Boolean, { nullable: true, default: () => false as const }),
|
|
325
|
+
}),
|
|
326
|
+
{ nullable: true },
|
|
327
|
+
),
|
|
328
|
+
}) {}
|
|
329
|
+
|
|
330
|
+
/**
|
|
331
|
+
* HTTP response information (URL, status code, optionally headers)
|
|
332
|
+
*/
|
|
333
|
+
export class ResponseInfo extends S.Class<ResponseInfo>('ResponseInfo')({
|
|
334
|
+
/**
|
|
335
|
+
* The URL that was fetched
|
|
336
|
+
*/
|
|
337
|
+
url: S.String,
|
|
338
|
+
/**
|
|
339
|
+
* HTTP status code of the response
|
|
340
|
+
*/
|
|
341
|
+
statusCode: S.Number,
|
|
342
|
+
/**
|
|
343
|
+
* Response headers from the fetch operation (keys are lower-cased HTTP header names)
|
|
344
|
+
*/
|
|
345
|
+
headers: S.optionalWith(S.Struct({}), { nullable: true }),
|
|
346
|
+
}) {}
|
|
347
|
+
|
|
348
|
+
/**
|
|
349
|
+
* Favicon or app icon metadata from <link> tags
|
|
350
|
+
*/
|
|
351
|
+
export class IconMeta extends S.Class<IconMeta>('IconMeta')({
|
|
352
|
+
/**
|
|
353
|
+
* Icon URL or path
|
|
354
|
+
*/
|
|
355
|
+
href: S.String,
|
|
356
|
+
/**
|
|
357
|
+
* Link relationship type
|
|
358
|
+
*/
|
|
359
|
+
rel: S.String,
|
|
360
|
+
/**
|
|
361
|
+
* MIME type of the icon
|
|
362
|
+
*/
|
|
363
|
+
type: S.optionalWith(S.String, { nullable: true }),
|
|
364
|
+
/**
|
|
365
|
+
* Icon dimensions
|
|
366
|
+
*/
|
|
367
|
+
sizes: S.optionalWith(S.String, { nullable: true }),
|
|
368
|
+
}) {}
|
|
369
|
+
|
|
370
|
+
/**
|
|
371
|
+
* Open Graph image with optional dimensions and alt text
|
|
372
|
+
*/
|
|
373
|
+
export class OgImage extends S.Class<OgImage>('OgImage')({
|
|
374
|
+
/**
|
|
375
|
+
* Image URL
|
|
376
|
+
*/
|
|
377
|
+
url: S.String,
|
|
378
|
+
/**
|
|
379
|
+
* Image width in pixels
|
|
380
|
+
*/
|
|
381
|
+
width: S.optionalWith(S.Number, { nullable: true }),
|
|
382
|
+
/**
|
|
383
|
+
* Image height in pixels
|
|
384
|
+
*/
|
|
385
|
+
height: S.optionalWith(S.Number, { nullable: true }),
|
|
386
|
+
/**
|
|
387
|
+
* Image alt text for accessibility
|
|
388
|
+
*/
|
|
389
|
+
alt: S.optionalWith(S.String, { nullable: true }),
|
|
390
|
+
}) {}
|
|
391
|
+
|
|
392
|
+
/**
|
|
393
|
+
* Open Graph protocol metadata for rich link previews
|
|
394
|
+
*/
|
|
395
|
+
export class OpenGraphMeta extends S.Class<OpenGraphMeta>('OpenGraphMeta')({
|
|
396
|
+
/**
|
|
397
|
+
* Open Graph title (og:title)
|
|
398
|
+
*/
|
|
399
|
+
title: S.optionalWith(S.String, { nullable: true }),
|
|
400
|
+
/**
|
|
401
|
+
* Open Graph description (og:description)
|
|
402
|
+
*/
|
|
403
|
+
description: S.optionalWith(S.String, { nullable: true }),
|
|
404
|
+
/**
|
|
405
|
+
* Open Graph type (og:type) - website, article, product, etc.
|
|
406
|
+
*/
|
|
407
|
+
type: S.optionalWith(S.String, { nullable: true }),
|
|
408
|
+
/**
|
|
409
|
+
* Canonical URL for the content (og:url)
|
|
410
|
+
*/
|
|
411
|
+
url: S.optionalWith(S.String, { nullable: true }),
|
|
412
|
+
/**
|
|
413
|
+
* Site name (og:site_name)
|
|
414
|
+
*/
|
|
415
|
+
siteName: S.optionalWith(S.String, { nullable: true }),
|
|
416
|
+
/**
|
|
417
|
+
* Locale in language_TERRITORY format (og:locale)
|
|
418
|
+
*/
|
|
419
|
+
locale: S.optionalWith(S.String, { nullable: true }),
|
|
420
|
+
/**
|
|
421
|
+
* Open Graph images (og:image and related properties)
|
|
422
|
+
*/
|
|
423
|
+
images: S.optionalWith(S.Array(OgImage), { nullable: true }),
|
|
424
|
+
}) {}
|
|
425
|
+
|
|
426
|
+
/**
|
|
427
|
+
* Twitter Card metadata for social sharing previews
|
|
428
|
+
*/
|
|
429
|
+
export class TwitterCardMeta extends S.Class<TwitterCardMeta>('TwitterCardMeta')({
|
|
430
|
+
/**
|
|
431
|
+
* Twitter card type (twitter:card)
|
|
432
|
+
*/
|
|
433
|
+
card: S.optionalWith(S.String, { nullable: true }),
|
|
434
|
+
/**
|
|
435
|
+
* Twitter @username of the website (twitter:site)
|
|
436
|
+
*/
|
|
437
|
+
site: S.optionalWith(S.String, { nullable: true }),
|
|
438
|
+
/**
|
|
439
|
+
* Twitter @username of content creator (twitter:creator)
|
|
440
|
+
*/
|
|
441
|
+
creator: S.optionalWith(S.String, { nullable: true }),
|
|
442
|
+
/**
|
|
443
|
+
* Title for Twitter card (twitter:title)
|
|
444
|
+
*/
|
|
445
|
+
title: S.optionalWith(S.String, { nullable: true }),
|
|
446
|
+
/**
|
|
447
|
+
* Description for Twitter card (twitter:description)
|
|
448
|
+
*/
|
|
449
|
+
description: S.optionalWith(S.String, { nullable: true }),
|
|
450
|
+
/**
|
|
451
|
+
* Image URL for Twitter card (twitter:image)
|
|
452
|
+
*/
|
|
453
|
+
image: S.optionalWith(S.String, { nullable: true }),
|
|
454
|
+
}) {}
|
|
455
|
+
|
|
456
|
+
/**
|
|
457
|
+
* Comprehensive metadata extracted from the page HTML head section
|
|
458
|
+
*/
|
|
459
|
+
export class PageMeta extends S.Class<PageMeta>('PageMeta')({
|
|
460
|
+
/**
|
|
461
|
+
* Page title from <title> tag
|
|
462
|
+
*/
|
|
463
|
+
title: S.optionalWith(S.String, { nullable: true }),
|
|
464
|
+
/**
|
|
465
|
+
* Meta description from <meta name="description">
|
|
466
|
+
*/
|
|
467
|
+
description: S.optionalWith(S.String, { nullable: true }),
|
|
468
|
+
/**
|
|
469
|
+
* Canonical URL from <link rel="canonical">
|
|
470
|
+
*/
|
|
471
|
+
canonicalUrl: S.optionalWith(S.String, { nullable: true }),
|
|
472
|
+
/**
|
|
473
|
+
* Page language from <html lang="...">
|
|
474
|
+
*/
|
|
475
|
+
language: S.optionalWith(S.String, { nullable: true }),
|
|
476
|
+
/**
|
|
477
|
+
* Character encoding from <meta charset="...">
|
|
478
|
+
*/
|
|
479
|
+
charset: S.optionalWith(S.String, { nullable: true }),
|
|
480
|
+
/**
|
|
481
|
+
* Primary favicon URL (first icon found)
|
|
482
|
+
*/
|
|
483
|
+
favicon: S.optionalWith(S.String, { nullable: true }),
|
|
484
|
+
/**
|
|
485
|
+
* All icon links (favicons, apple-touch-icons, etc.)
|
|
486
|
+
*/
|
|
487
|
+
icons: S.optionalWith(S.Array(IconMeta), { nullable: true }),
|
|
488
|
+
/**
|
|
489
|
+
* Open Graph metadata from og:* meta tags
|
|
490
|
+
*/
|
|
491
|
+
openGraph: S.optionalWith(OpenGraphMeta, { nullable: true }),
|
|
492
|
+
/**
|
|
493
|
+
* Twitter Card metadata from twitter:* meta tags
|
|
494
|
+
*/
|
|
495
|
+
twitter: S.optionalWith(TwitterCardMeta, { nullable: true }),
|
|
496
|
+
}) {}
|
|
497
|
+
|
|
498
|
+
/**
|
|
499
|
+
* Contains the extracted content in the formats specified by the select configuration
|
|
500
|
+
*/
|
|
501
|
+
export class FetchData extends S.Class<FetchData>('FetchData')({
|
|
502
|
+
response: ResponseInfo,
|
|
503
|
+
/**
|
|
504
|
+
* Page metadata extracted from HTML head (title, description, Open Graph, Twitter Card, icons)
|
|
505
|
+
*/
|
|
506
|
+
meta: S.optionalWith(PageMeta, { nullable: true }),
|
|
507
|
+
/**
|
|
508
|
+
* The HTML content of the fetched page
|
|
509
|
+
*/
|
|
510
|
+
html: S.optionalWith(S.String, { nullable: true }),
|
|
511
|
+
/**
|
|
512
|
+
* The markdown-formatted content extracted from the page
|
|
513
|
+
*/
|
|
514
|
+
markdown: S.optionalWith(S.String, { nullable: true }),
|
|
515
|
+
/**
|
|
516
|
+
* Base64-encoded data URI of the screenshot image
|
|
517
|
+
*/
|
|
518
|
+
screenshot: S.optionalWith(S.String, { nullable: true }),
|
|
519
|
+
/**
|
|
520
|
+
* AI-generated summary of the page content
|
|
521
|
+
*/
|
|
522
|
+
summary: S.optionalWith(S.String, { nullable: true }),
|
|
523
|
+
/**
|
|
524
|
+
* Relevant snippets extracted from the page based on the search query
|
|
525
|
+
*/
|
|
526
|
+
snippets: S.optionalWith(
|
|
527
|
+
S.Array(
|
|
528
|
+
S.Struct({
|
|
529
|
+
/**
|
|
530
|
+
* Type identifier for TextPart compatibility
|
|
531
|
+
*/
|
|
532
|
+
type: S.optionalWith(S.Literal('text'), { nullable: true, default: () => 'text' as const }),
|
|
533
|
+
/**
|
|
534
|
+
* The text content of the snippet
|
|
535
|
+
*/
|
|
536
|
+
text: S.String,
|
|
537
|
+
/**
|
|
538
|
+
* Relevance score from the reranker (0-1)
|
|
539
|
+
*/
|
|
540
|
+
score: S.Number,
|
|
541
|
+
/**
|
|
542
|
+
* Original chunk index
|
|
543
|
+
*/
|
|
544
|
+
index: S.Number,
|
|
545
|
+
}),
|
|
546
|
+
),
|
|
547
|
+
{ nullable: true },
|
|
548
|
+
),
|
|
549
|
+
}) {}
|
|
550
|
+
|
|
551
|
+
/**
|
|
552
|
+
* Complete response from a fetch operation
|
|
553
|
+
*/
|
|
554
|
+
export class FetchResult extends S.Class<FetchResult>('FetchResult')({
|
|
555
|
+
data: FetchData,
|
|
556
|
+
}) {}
|
|
557
|
+
|
|
558
|
+
export class FetchErrorTag extends S.Literal('FetchError') {}
|
|
559
|
+
|
|
560
|
+
export class FetchError extends S.Class<FetchError>('FetchError')({
|
|
561
|
+
_tag: FetchErrorTag,
|
|
562
|
+
}) {}
|
|
563
|
+
|
|
564
|
+
/**
|
|
565
|
+
* Request parameters for fetching and converting web page content to markdown format
|
|
566
|
+
*/
|
|
567
|
+
export class MarkdownRequest extends S.Class<MarkdownRequest>('MarkdownRequest')({
|
|
568
|
+
/**
|
|
569
|
+
* The URL to fetch markdown content from
|
|
570
|
+
*/
|
|
571
|
+
url: S.String,
|
|
572
|
+
/**
|
|
573
|
+
* Configuration options for browser behavior during the fetch
|
|
574
|
+
*/
|
|
575
|
+
browserConfig: S.optionalWith(
|
|
576
|
+
S.Struct({
|
|
577
|
+
/**
|
|
578
|
+
* Whether to scroll the entire page to capture lazy-loaded content
|
|
579
|
+
*/
|
|
580
|
+
scrollFullPage: S.optionalWith(S.Boolean, { nullable: true, default: () => false as const }),
|
|
581
|
+
}),
|
|
582
|
+
{ nullable: true },
|
|
583
|
+
),
|
|
584
|
+
}) {}
|
|
585
|
+
|
|
586
|
+
export class InvalidAssetHashTag extends S.Literal('InvalidAssetHash') {}
|
|
587
|
+
|
|
588
|
+
export class InvalidAssetHash extends S.Class<InvalidAssetHash>('InvalidAssetHash')({
|
|
589
|
+
message: S.String,
|
|
590
|
+
_tag: InvalidAssetHashTag,
|
|
591
|
+
}) {}
|
|
592
|
+
|
|
593
|
+
export class AssetNotFoundInHarTag extends S.Literal('AssetNotFoundInHar') {}
|
|
594
|
+
|
|
595
|
+
export class AssetNotFoundInHar extends S.Class<AssetNotFoundInHar>('AssetNotFoundInHar')({
|
|
596
|
+
snapshotDir: S.String,
|
|
597
|
+
assetUrl: S.String,
|
|
598
|
+
_tag: AssetNotFoundInHarTag,
|
|
599
|
+
}) {}
|
|
600
|
+
|
|
601
|
+
export class HarNotFoundTag extends S.Literal('HarNotFound') {}
|
|
602
|
+
|
|
603
|
+
export class HarNotFound extends S.Class<HarNotFound>('HarNotFound')({
|
|
604
|
+
snapshotDir: S.String,
|
|
605
|
+
_tag: HarNotFoundTag,
|
|
606
|
+
}) {}
|
|
607
|
+
|
|
608
|
+
export class AssetFetchErrorTag extends S.Literal('AssetFetchError') {}
|
|
609
|
+
|
|
610
|
+
export class AssetFetchError extends S.Class<AssetFetchError>('AssetFetchError')({
|
|
611
|
+
snapshotDir: S.String,
|
|
612
|
+
assetUrl: S.String,
|
|
613
|
+
_tag: AssetFetchErrorTag,
|
|
614
|
+
}) {}
|
|
615
|
+
|
|
616
|
+
export class LocalAssetsGetAsset500 extends S.Union(
|
|
617
|
+
InvalidAssetHash,
|
|
618
|
+
AssetNotFoundInHar,
|
|
619
|
+
HarNotFound,
|
|
620
|
+
AssetFetchError,
|
|
621
|
+
) {}
|
|
622
|
+
|
|
623
|
+
export const make = (
|
|
624
|
+
httpClient: HttpClient.HttpClient,
|
|
625
|
+
options: {
|
|
626
|
+
readonly transformClient?: ((client: HttpClient.HttpClient) => Effect.Effect<HttpClient.HttpClient>) | undefined
|
|
627
|
+
} = {},
|
|
628
|
+
): Client => {
|
|
629
|
+
const unexpectedStatus = (response: HttpClientResponse.HttpClientResponse) =>
|
|
630
|
+
Effect.flatMap(
|
|
631
|
+
Effect.orElseSucceed(response.json, () => 'Unexpected status code'),
|
|
632
|
+
(description) =>
|
|
633
|
+
Effect.fail(
|
|
634
|
+
new HttpClientError.ResponseError({
|
|
635
|
+
request: response.request,
|
|
636
|
+
response,
|
|
637
|
+
reason: 'StatusCode',
|
|
638
|
+
description: typeof description === 'string' ? description : JSON.stringify(description),
|
|
639
|
+
}),
|
|
640
|
+
),
|
|
641
|
+
)
|
|
642
|
+
const withResponse: <A, E>(
|
|
643
|
+
f: (response: HttpClientResponse.HttpClientResponse) => Effect.Effect<A, E>,
|
|
644
|
+
) => (request: HttpClientRequest.HttpClientRequest) => Effect.Effect<any, any> = options.transformClient
|
|
645
|
+
? (f) => (request) =>
|
|
646
|
+
Effect.flatMap(
|
|
647
|
+
Effect.flatMap(options.transformClient!(httpClient), (client) => client.execute(request)),
|
|
648
|
+
f,
|
|
649
|
+
)
|
|
650
|
+
: (f) => (request) => Effect.flatMap(httpClient.execute(request), f)
|
|
651
|
+
const decodeSuccess =
|
|
652
|
+
<A, I, R>(schema: S.Schema<A, I, R>) =>
|
|
653
|
+
(response: HttpClientResponse.HttpClientResponse) =>
|
|
654
|
+
HttpClientResponse.schemaBodyJson(schema)(response)
|
|
655
|
+
const decodeError =
|
|
656
|
+
<const Tag extends string, A, I, R>(tag: Tag, schema: S.Schema<A, I, R>) =>
|
|
657
|
+
(response: HttpClientResponse.HttpClientResponse) =>
|
|
658
|
+
Effect.flatMap(HttpClientResponse.schemaBodyJson(schema)(response), (cause) =>
|
|
659
|
+
Effect.fail(ClientError(tag, cause, response)),
|
|
660
|
+
)
|
|
661
|
+
return {
|
|
662
|
+
httpClient,
|
|
663
|
+
assetsGetByUrl: (browserSessionId, options) =>
|
|
664
|
+
HttpClientRequest.get(`/v1/assets/${browserSessionId}/asset`).pipe(
|
|
665
|
+
HttpClientRequest.setUrlParams({ url: options?.['url'] as any }),
|
|
666
|
+
withResponse(
|
|
667
|
+
HttpClientResponse.matchStatus({
|
|
668
|
+
'400': decodeError('HttpApiDecodeError', HttpApiDecodeError),
|
|
669
|
+
'401': decodeError('UnauthorizedAccess', UnauthorizedAccess),
|
|
670
|
+
'404': decodeError('AssetsGetByUrl404', AssetsGetByUrl404),
|
|
671
|
+
'500': decodeError('AssetsGetByUrl500', AssetsGetByUrl500),
|
|
672
|
+
'502': decodeError('CurlError', CurlError),
|
|
673
|
+
'429': () => Effect.void,
|
|
674
|
+
orElse: unexpectedStatus,
|
|
675
|
+
}),
|
|
676
|
+
),
|
|
677
|
+
),
|
|
678
|
+
assetsList: (browserSessionId) =>
|
|
679
|
+
HttpClientRequest.get(`/v1/assets/${browserSessionId}`).pipe(
|
|
680
|
+
withResponse(
|
|
681
|
+
HttpClientResponse.matchStatus({
|
|
682
|
+
'2xx': decodeSuccess(AssetListResponse),
|
|
683
|
+
'400': decodeError('HttpApiDecodeError', HttpApiDecodeError),
|
|
684
|
+
'401': decodeError('UnauthorizedAccess', UnauthorizedAccess),
|
|
685
|
+
'404': decodeError('AssetNotFound', AssetNotFound),
|
|
686
|
+
'500': decodeError('AssetsList500', AssetsList500),
|
|
687
|
+
'429': () => Effect.void,
|
|
688
|
+
orElse: unexpectedStatus,
|
|
689
|
+
}),
|
|
690
|
+
),
|
|
691
|
+
),
|
|
692
|
+
fetch: (options) =>
|
|
693
|
+
HttpClientRequest.post(`/v1/fetch`).pipe(
|
|
694
|
+
HttpClientRequest.bodyUnsafeJson(options),
|
|
695
|
+
withResponse(
|
|
696
|
+
HttpClientResponse.matchStatus({
|
|
697
|
+
'2xx': decodeSuccess(FetchResult),
|
|
698
|
+
'400': decodeError('HttpApiDecodeError', HttpApiDecodeError),
|
|
699
|
+
'401': decodeError('UnauthorizedAccess', UnauthorizedAccess),
|
|
700
|
+
'500': decodeError('FetchError', FetchError),
|
|
701
|
+
'429': () => Effect.void,
|
|
702
|
+
orElse: unexpectedStatus,
|
|
703
|
+
}),
|
|
704
|
+
),
|
|
705
|
+
),
|
|
706
|
+
markdown: (options) =>
|
|
707
|
+
HttpClientRequest.post(`/v1/fetch/markdown`).pipe(
|
|
708
|
+
HttpClientRequest.bodyUnsafeJson(options),
|
|
709
|
+
withResponse(
|
|
710
|
+
HttpClientResponse.matchStatus({
|
|
711
|
+
'400': decodeError('HttpApiDecodeError', HttpApiDecodeError),
|
|
712
|
+
'401': decodeError('UnauthorizedAccess', UnauthorizedAccess),
|
|
713
|
+
'500': decodeError('FetchError', FetchError),
|
|
714
|
+
'429': () => Effect.void,
|
|
715
|
+
orElse: unexpectedStatus,
|
|
716
|
+
}),
|
|
717
|
+
),
|
|
718
|
+
),
|
|
719
|
+
localAssetsGetAsset: (assetHash) =>
|
|
720
|
+
HttpClientRequest.get(`/local-asset/${assetHash}`).pipe(
|
|
721
|
+
withResponse(
|
|
722
|
+
HttpClientResponse.matchStatus({
|
|
723
|
+
'400': decodeError('HttpApiDecodeError', HttpApiDecodeError),
|
|
724
|
+
'500': decodeError('LocalAssetsGetAsset500', LocalAssetsGetAsset500),
|
|
725
|
+
'204': () => Effect.void,
|
|
726
|
+
'429': () => Effect.void,
|
|
727
|
+
orElse: unexpectedStatus,
|
|
728
|
+
}),
|
|
729
|
+
),
|
|
730
|
+
),
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
export interface Client {
|
|
735
|
+
readonly httpClient: HttpClient.HttpClient
|
|
736
|
+
readonly assetsGetByUrl: (
|
|
737
|
+
browserSessionId: string,
|
|
738
|
+
options: typeof AssetsGetByUrlParams.Encoded,
|
|
739
|
+
) => Effect.Effect<
|
|
740
|
+
void,
|
|
741
|
+
| HttpClientError.HttpClientError
|
|
742
|
+
| ParseError
|
|
743
|
+
| ClientError<'HttpApiDecodeError', typeof HttpApiDecodeError.Type>
|
|
744
|
+
| ClientError<'UnauthorizedAccess', typeof UnauthorizedAccess.Type>
|
|
745
|
+
| ClientError<'AssetsGetByUrl404', typeof AssetsGetByUrl404.Type>
|
|
746
|
+
| ClientError<'AssetsGetByUrl500', typeof AssetsGetByUrl500.Type>
|
|
747
|
+
| ClientError<'CurlError', typeof CurlError.Type>
|
|
748
|
+
>
|
|
749
|
+
readonly assetsList: (
|
|
750
|
+
browserSessionId: string,
|
|
751
|
+
) => Effect.Effect<
|
|
752
|
+
typeof AssetListResponse.Type,
|
|
753
|
+
| HttpClientError.HttpClientError
|
|
754
|
+
| ParseError
|
|
755
|
+
| ClientError<'HttpApiDecodeError', typeof HttpApiDecodeError.Type>
|
|
756
|
+
| ClientError<'UnauthorizedAccess', typeof UnauthorizedAccess.Type>
|
|
757
|
+
| ClientError<'AssetNotFound', typeof AssetNotFound.Type>
|
|
758
|
+
| ClientError<'AssetsList500', typeof AssetsList500.Type>
|
|
759
|
+
>
|
|
760
|
+
/**
|
|
761
|
+
* Extract content from a web page as HTML, markdown, and/or screenshots. Handles complex rendering and bot protection.
|
|
762
|
+
*/
|
|
763
|
+
readonly fetch: (
|
|
764
|
+
options: typeof FetchRequest.Encoded,
|
|
765
|
+
) => Effect.Effect<
|
|
766
|
+
typeof FetchResult.Type,
|
|
767
|
+
| HttpClientError.HttpClientError
|
|
768
|
+
| ParseError
|
|
769
|
+
| ClientError<'HttpApiDecodeError', typeof HttpApiDecodeError.Type>
|
|
770
|
+
| ClientError<'UnauthorizedAccess', typeof UnauthorizedAccess.Type>
|
|
771
|
+
| ClientError<'FetchError', typeof FetchError.Type>
|
|
772
|
+
>
|
|
773
|
+
/**
|
|
774
|
+
* Extract and convert web page content to markdown. Returns clean, structured content for LLMs.
|
|
775
|
+
*/
|
|
776
|
+
readonly markdown: (
|
|
777
|
+
options: typeof MarkdownRequest.Encoded,
|
|
778
|
+
) => Effect.Effect<
|
|
779
|
+
void,
|
|
780
|
+
| HttpClientError.HttpClientError
|
|
781
|
+
| ParseError
|
|
782
|
+
| ClientError<'HttpApiDecodeError', typeof HttpApiDecodeError.Type>
|
|
783
|
+
| ClientError<'UnauthorizedAccess', typeof UnauthorizedAccess.Type>
|
|
784
|
+
| ClientError<'FetchError', typeof FetchError.Type>
|
|
785
|
+
>
|
|
786
|
+
readonly localAssetsGetAsset: (
|
|
787
|
+
assetHash: string,
|
|
788
|
+
) => Effect.Effect<
|
|
789
|
+
void,
|
|
790
|
+
| HttpClientError.HttpClientError
|
|
791
|
+
| ParseError
|
|
792
|
+
| ClientError<'HttpApiDecodeError', typeof HttpApiDecodeError.Type>
|
|
793
|
+
| ClientError<'LocalAssetsGetAsset500', typeof LocalAssetsGetAsset500.Type>
|
|
794
|
+
>
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
export interface ClientError<Tag extends string, E> {
|
|
798
|
+
readonly _tag: Tag
|
|
799
|
+
readonly request: HttpClientRequest.HttpClientRequest
|
|
800
|
+
readonly response: HttpClientResponse.HttpClientResponse
|
|
801
|
+
readonly cause: E
|
|
802
|
+
}
|
|
803
|
+
|
|
804
|
+
class ClientErrorImpl extends Data.Error<{
|
|
805
|
+
_tag: string
|
|
806
|
+
cause: any
|
|
807
|
+
request: HttpClientRequest.HttpClientRequest
|
|
808
|
+
response: HttpClientResponse.HttpClientResponse
|
|
809
|
+
}> {}
|
|
810
|
+
|
|
811
|
+
export const ClientError = <Tag extends string, E>(
|
|
812
|
+
tag: Tag,
|
|
813
|
+
cause: E,
|
|
814
|
+
response: HttpClientResponse.HttpClientResponse,
|
|
815
|
+
): ClientError<Tag, E> =>
|
|
816
|
+
new ClientErrorImpl({
|
|
817
|
+
_tag: tag,
|
|
818
|
+
cause,
|
|
819
|
+
response,
|
|
820
|
+
request: response.request,
|
|
821
|
+
}) as any
|