@expandai/sdk 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +130 -4
- package/build/dts/AppendixMarkers.d.ts +30 -0
- package/build/dts/AppendixMarkers.d.ts.map +1 -0
- package/build/dts/Client.d.ts +41 -6
- package/build/dts/Client.d.ts.map +1 -1
- package/build/dts/Crawls.d.ts +568 -0
- package/build/dts/Crawls.d.ts.map +1 -0
- package/build/dts/EnvProxyHttpClient.d.ts +28 -0
- package/build/dts/EnvProxyHttpClient.d.ts.map +1 -0
- package/build/dts/FetchJson.d.ts +444 -37
- package/build/dts/FetchJson.d.ts.map +1 -1
- package/build/dts/Generated.d.ts +1545 -788
- package/build/dts/Generated.d.ts.map +1 -1
- package/build/dts/HttpClientHelpers.d.ts +13 -0
- package/build/dts/HttpClientHelpers.d.ts.map +1 -1
- package/build/dts/Internal.d.ts +6 -4
- package/build/dts/Internal.d.ts.map +1 -1
- package/build/dts/ParseErrorBody.d.ts +26 -0
- package/build/dts/ParseErrorBody.d.ts.map +1 -0
- package/build/dts/Playground.d.ts +20 -0
- package/build/dts/Playground.d.ts.map +1 -1
- package/build/dts/RetryAfter.d.ts +40 -0
- package/build/dts/RetryAfter.d.ts.map +1 -0
- package/build/dts/Service.d.ts +94 -45
- package/build/dts/Service.d.ts.map +1 -1
- package/build/dts/StreamPage.d.ts +10 -0
- package/build/dts/StreamPage.d.ts.map +1 -0
- package/build/dts/index.d.ts +9 -2
- package/build/dts/index.d.ts.map +1 -1
- package/build/esm/AppendixMarkers.js +307 -0
- package/build/esm/AppendixMarkers.js.map +1 -0
- package/build/esm/Client.js +96 -15
- package/build/esm/Client.js.map +1 -1
- package/build/esm/Crawls.js +786 -0
- package/build/esm/Crawls.js.map +1 -0
- package/build/esm/EnvProxyHttpClient.js +55 -0
- package/build/esm/EnvProxyHttpClient.js.map +1 -0
- package/build/esm/FetchJson.js +48 -22
- package/build/esm/FetchJson.js.map +1 -1
- package/build/esm/Generated.js +540 -112
- package/build/esm/Generated.js.map +1 -1
- package/build/esm/HttpClientHelpers.js +36 -1
- package/build/esm/HttpClientHelpers.js.map +1 -1
- package/build/esm/Internal.js +140 -33
- package/build/esm/Internal.js.map +1 -1
- package/build/esm/ParseErrorBody.js +41 -0
- package/build/esm/ParseErrorBody.js.map +1 -0
- package/build/esm/Playground.js +38 -0
- package/build/esm/Playground.js.map +1 -1
- package/build/esm/RetryAfter.js +104 -0
- package/build/esm/RetryAfter.js.map +1 -0
- package/build/esm/Service.js +96 -17
- package/build/esm/Service.js.map +1 -1
- package/build/esm/StreamPage.js +46 -0
- package/build/esm/StreamPage.js.map +1 -0
- package/build/esm/index.js +5 -1
- package/build/esm/index.js.map +1 -1
- package/package.json +8 -2
- package/src/AppendixMarkers.ts +370 -0
- package/src/Client.ts +131 -16
- package/src/Crawls.ts +1387 -0
- package/src/EnvProxyHttpClient.ts +71 -0
- package/src/FetchJson.ts +63 -25
- package/src/Generated.ts +738 -162
- package/src/HttpClientHelpers.ts +48 -1
- package/src/Internal.ts +163 -105
- package/src/ParseErrorBody.ts +42 -0
- package/src/Playground.ts +48 -0
- package/src/RetryAfter.ts +138 -0
- package/src/Service.ts +208 -25
- package/src/StreamPage.ts +47 -0
- package/src/index.ts +29 -3
package/README.md
CHANGED
|
@@ -6,6 +6,8 @@ Built on [Effect](https://effect.website), with both a class-based client for pr
|
|
|
6
6
|
|
|
7
7
|
## Install
|
|
8
8
|
|
|
9
|
+
Requires Node.js 20.3 or newer.
|
|
10
|
+
|
|
9
11
|
```sh
|
|
10
12
|
npm install @expandai/sdk
|
|
11
13
|
# or
|
|
@@ -14,6 +16,14 @@ bun add @expandai/sdk
|
|
|
14
16
|
pnpm add @expandai/sdk
|
|
15
17
|
```
|
|
16
18
|
|
|
19
|
+
Internal staging builds are available without changing the production `latest` tag:
|
|
20
|
+
|
|
21
|
+
```sh
|
|
22
|
+
npm install @expandai/sdk@staging
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Pass the staging API URL through the client's `baseUrl`; selecting the npm tag does not change the default production endpoint.
|
|
26
|
+
|
|
17
27
|
## Authentication
|
|
18
28
|
|
|
19
29
|
Set `EXPAND_API_KEY` in your environment, or pass `apiKey` explicitly when constructing the client.
|
|
@@ -31,10 +41,16 @@ import { ExpandClient } from '@expandai/sdk'
|
|
|
31
41
|
|
|
32
42
|
const expand = new ExpandClient()
|
|
33
43
|
|
|
34
|
-
|
|
35
|
-
|
|
44
|
+
try {
|
|
45
|
+
const result = await expand.fetch({ url: 'https://example.com' })
|
|
46
|
+
console.log(result)
|
|
47
|
+
} finally {
|
|
48
|
+
await expand.close()
|
|
49
|
+
}
|
|
36
50
|
```
|
|
37
51
|
|
|
52
|
+
Reuse a client for the lifetime of your application, then call `close()` during shutdown. Closing aborts active requests, ends Crawl iterators and polling, and releases this client's connections without closing another client's pool. Repeated closes are safe; later requests on a closed client reject. Effect service consumers retain the existing scoped-layer cleanup.
|
|
53
|
+
|
|
38
54
|
Constructor options:
|
|
39
55
|
|
|
40
56
|
```ts
|
|
@@ -42,7 +58,7 @@ new ExpandClient({
|
|
|
42
58
|
apiKey: 'xpnd_...', // defaults to process.env.EXPAND_API_KEY
|
|
43
59
|
baseUrl: 'https://api...', // defaults to https://api.expand.ai
|
|
44
60
|
timeoutMs: 60_000, // total wall-clock budget incl. retries
|
|
45
|
-
maxRetries: 2, // retries
|
|
61
|
+
maxRetries: 2, // retries transient failures; see "What gets retried"
|
|
46
62
|
})
|
|
47
63
|
```
|
|
48
64
|
|
|
@@ -76,7 +92,7 @@ ExpandService.layer({
|
|
|
76
92
|
apiKey: 'xpnd_...', // defaults to process.env.EXPAND_API_KEY
|
|
77
93
|
baseUrl: 'https://api...', // defaults to https://api.expand.ai
|
|
78
94
|
timeoutMs: 60_000, // total wall-clock budget incl. retries
|
|
79
|
-
maxRetries: 2, // retries
|
|
95
|
+
maxRetries: 2, // retries transient failures; see "What gets retried"
|
|
80
96
|
})
|
|
81
97
|
```
|
|
82
98
|
|
|
@@ -87,6 +103,95 @@ Typed errors in the `Effect` channel:
|
|
|
87
103
|
- `ExpandConnectionError` — network failure
|
|
88
104
|
- `ExpandTimeoutError` — request timed out
|
|
89
105
|
|
|
106
|
+
## Crawl and capture projections
|
|
107
|
+
|
|
108
|
+
Creation and run-command replays require a caller-persisted idempotency key. One-off crawls close permanently when done, failed, or canceled; create a new crawl to run again. Events carry their own resume offset.
|
|
109
|
+
|
|
110
|
+
Replaying a creation key returns the original ordered Crawl IDs, even if a member has since expired. Reading an expired member returns 404; replay never recreates it or starts a replacement run.
|
|
111
|
+
|
|
112
|
+
### How a crawl finds pages
|
|
113
|
+
|
|
114
|
+
A crawl starts at each source URL and stays on that URL's registrable domain and its subdomains.
|
|
115
|
+
|
|
116
|
+
- **`maxDepth` counts clicks from your URL.** `0` is the URL alone. `1` adds the pages it links to and the pages in the site's sitemap. `2` adds the pages those link to, and so on. Pages found in the sitemap or by search count as one click, because they were not on your list. A `kind: 'sitemap'` source is the exception: the URLs it lists are yours, so they start at depth 0. The default, 50, is the whole site up to `maxPages`.
|
|
117
|
+
- **`include` decides what ends up in the results, not where the crawler goes.** To reach `/p/**` from the homepage, the crawler opens the pages that link there, such as `/` and `/damen`, and follows their links. It does not keep those pages, and they do not count toward `maxPages`.
|
|
118
|
+
- **`exclude` keeps the crawler out.** A matching page is never opened, so nothing behind it is found through it.
|
|
119
|
+
- **`sync.freshness` is how often every page is checked for changes**, for example `24h`. How each check is made is up to the crawler.
|
|
120
|
+
|
|
121
|
+
`changePolicy.ignore` removes matching subtrees from change comparison; `changePolicy.volatile` replaces them with stable markers. Both leave saved page content unchanged. Selectors support static tags, classes, IDs, attributes, combinators and structural pseudo-classes such as `:not`, `:has` and `:nth-child`. Interaction states such as `:hover`, pseudo-elements, named namespace prefixes, nested `:has` and `:nth-child(... of ...)` are rejected. Matching includes captured shadow subtrees and uses case-sensitive class and ID values even on quirks-mode pages.
|
|
122
|
+
|
|
123
|
+
```ts
|
|
124
|
+
import { Expand } from '@expandai/sdk'
|
|
125
|
+
|
|
126
|
+
const expand = new Expand({ apiKey: process.env.EXPAND_API_KEY })
|
|
127
|
+
const [crawl] = await expand.crawls.create(
|
|
128
|
+
{ sources: ['https://example.com/products'], formats: ['markdown', 'stateJson'] },
|
|
129
|
+
{ idempotencyKey: 'catalog-import-2026-09' },
|
|
130
|
+
)
|
|
131
|
+
if (!crawl) throw new Error('Expected a Crawl')
|
|
132
|
+
|
|
133
|
+
for await (const event of crawl.events()) {
|
|
134
|
+
if (event.type !== 'page.added' && event.type !== 'page.changed') continue
|
|
135
|
+
const capture = expand.captures(event.captureId)
|
|
136
|
+
const state = event.page.stateJson ?? await capture.stateJson()
|
|
137
|
+
for (const evidence of state.evidence) {
|
|
138
|
+
const body = evidence.body !== undefined ? evidence.body : (evidence.bodyUrl ? await expand.fetchJson(evidence.bodyUrl) : undefined)
|
|
139
|
+
// Process body, then persist event.offset in your application.
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
`captures(id)` also exposes `get()`, `markdown()`, `html()`, `evidence(key)` and `playgroundUrl()`. These read stored captures, not fresh pages. JSON body URLs must belong to the configured API origin and cannot redirect. Projection reads accept `signal` and `timeoutMs`; they do not perform automatic retries.
|
|
145
|
+
|
|
146
|
+
Catalog and version lists support individual items, pages, or one awaited page:
|
|
147
|
+
|
|
148
|
+
```ts
|
|
149
|
+
const list = crawl.pages({ limit: 100, status: 'live' })
|
|
150
|
+
const first = await list // { items, nextCursor? }
|
|
151
|
+
for await (const page of list.pages()) { /* page.items */ }
|
|
152
|
+
for await (const item of list) { /* item.id, item.url */ }
|
|
153
|
+
const bounded = await list.collect({ limit: 200 })
|
|
154
|
+
|
|
155
|
+
const versions = expand.pages('page-id').versions({ limit: 20 })
|
|
156
|
+
const diff = await expand.pages('page-id').diff({ from: 'capture-before', to: 'capture-after' })
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
A list instance caches its first page. Stop iteration with `break`, or cancel reads through `signal`. `crawl.page(id)` and `crawl.pageByUrl(url)` read the current catalog projection. Page-version lists use `before` or `at` for historical reads.
|
|
160
|
+
|
|
161
|
+
`events({ from: '-1' })` replays retained history, including changes, removals, and failures. It does not rebuild the current catalog. For a full catalog import or recovery after `CrawlOffsetExpiredError`, use the pinned snapshot:
|
|
162
|
+
|
|
163
|
+
```ts
|
|
164
|
+
for await (const batch of crawl.snapshot().pages()) {
|
|
165
|
+
// Apply batch.items, then persist batch.nextOffset.
|
|
166
|
+
// When batch.upToDate is true, that offset is the boundary to resume with crawl.events({ from }).
|
|
167
|
+
}
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
`snapshot({ from: savedSnapshotCursor })` resumes an unfinished snapshot. Snapshot cursors belong to `/snapshot`; an older bootstrap checkpoint submitted to `/events` returns 410. Event offsets are opaque and sortable: copy them exactly rather than converting them to numbers. One-off event and activity iterators stop at stream EOF. Recurring reads return when caught up unless `live: true` is set.
|
|
171
|
+
|
|
172
|
+
Event reads also expose bounded server pages. Awaiting an event read returns one page, including an empty caught-up page, and closes that read. Resume using its `nextOffset`. Event reads do not cache that first page across separate invocations.
|
|
173
|
+
|
|
174
|
+
Each non-live event request and control request honors the client's `timeoutMs` across its retries and `maxRetries` limit. Explicit `live: true` reads reconnect indefinitely on transient failures until canceled, returned from, or closed with the client. A server refusal marked `retryable: false` stops either mode. `done()` polls until completion or cancellation; its finite `pollIntervalMs` must be no greater than 2147483647, and values below 50 are clamped to 50ms.
|
|
175
|
+
|
|
176
|
+
Crawl API refusals extend `ExpandClientApiError` and `ExpandClientError`, with `status`, `body`, `code`, and `retryable` available. `CrawlOffsetExpiredError` remains the specific 410 error. Local validation errors extend `ExpandClientError` through `CrawlClientError`.
|
|
177
|
+
|
|
178
|
+
Effect users can call `yield* expand.crawls.create(request, { idempotencyKey })` through `ExpandService`, then use `status`, `update`, `cancel`, or `run` on the returned resources. These control methods share the service's HTTP client, retries, and typed `ExpandError` channel. Event iterators, `done()`, and capture projections currently use the Promise interface; the Effect service does not yet mirror those helpers.
|
|
179
|
+
|
|
180
|
+
Because these reads are thenable, `Promise.resolve`, `Promise.all`, or returning one from an `async` function also starts that one-page read. Live page iteration exposes empty long-poll timeout pages; item iteration hides them.
|
|
181
|
+
|
|
182
|
+
```ts
|
|
183
|
+
const batch = await crawl.events({ from: savedOffset })
|
|
184
|
+
if (batch) {
|
|
185
|
+
for (const event of batch.items) { /* process event */ }
|
|
186
|
+
// Persist batch.nextOffset after processing the whole batch.
|
|
187
|
+
}
|
|
188
|
+
for await (const batch of crawl.events({ live: true }).pages()) {
|
|
189
|
+
// batch.crawlId identifies the partition, including in merged reads.
|
|
190
|
+
}
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
Merged page reads take `from: { [crawlId]: offset }`, just like merged item reads. Each returned batch belongs to one Crawl and carries that Crawl's `nextOffset`; never reuse it for a different Crawl.
|
|
194
|
+
|
|
90
195
|
## Batched fetches
|
|
91
196
|
|
|
92
197
|
`batched` kicks off a job and returns immediately with an `id`. Poll `getBatched(id)` until `batchedStatus` is no longer `RUNNING` or `QUEUED`.
|
|
@@ -109,6 +214,8 @@ while (status.batchedStatus === 'RUNNING' || status.batchedStatus === 'QUEUED')
|
|
|
109
214
|
console.log(status.results)
|
|
110
215
|
```
|
|
111
216
|
|
|
217
|
+
Successful results include Markdown at `data.markdown` and State JSON at `data.json` by default when extraction succeeds.
|
|
218
|
+
|
|
112
219
|
Each result has a persisted `status` (`QUEUED`, `RUNNING`, `SUCCEEDED`, `FAILED`, or `CANCELLED`) alongside its `data`.
|
|
113
220
|
|
|
114
221
|
### Effect service
|
|
@@ -136,6 +243,25 @@ NodeRuntime.runMain(program.pipe(Effect.provide(ExpandService.layer({ apiKey: pr
|
|
|
136
243
|
|
|
137
244
|
The SDK generates an idempotency key and reuses it for automatic retries when one is omitted. Supply a stable `idempotencyKey` to deduplicate separate calls or calls made after a process restart. Reusing a key with a different payload returns `409 BatchedIdempotencyConflict`.
|
|
138
245
|
|
|
246
|
+
## What gets retried
|
|
247
|
+
|
|
248
|
+
Not every 5xx is transient. Some of them are verdicts about the page you asked for — the browser committed its own error document, or the page could not be captured after the capture retries were spent — and repeating the request re-runs a full capture to reach the same answer, on your timeout budget and your quota.
|
|
249
|
+
|
|
250
|
+
So the server states the answer rather than leaving it to a status-code rule. Error bodies whose status is ambiguous (`429`, `500`, `502`, `503`, `504`, `529`) carry a `retryable` boolean, and the SDK follows it:
|
|
251
|
+
|
|
252
|
+
| Response | Retried |
|
|
253
|
+
| --- | --- |
|
|
254
|
+
| Timeout, connection failure | yes |
|
|
255
|
+
| `retryable: true` (e.g. `429 TooManyRequests`, `503 ServiceUnavailable`, `529 FetchCapacityTimeout`) | yes |
|
|
256
|
+
| `retryable: false` (e.g. `502 FetchNavigationFailed`, `502 FetchCaptureFailed`, `504 FetchCaptureTimeout`) | no |
|
|
257
|
+
| A verdict tag with no `retryable` field — `409 BatchedIdempotencyConflict`, `409 BatchedCancellationConflict`, `502 FetchNavigationFailed`, `502 FetchCaptureFailed`, `504 FetchCaptureTimeout` | no |
|
|
258
|
+
| `408`, `429`, other `5xx` with no `retryable` field | yes |
|
|
259
|
+
| Any other non-2xx | no |
|
|
260
|
+
|
|
261
|
+
Because the verdict travels in the body, a newly-terminal error class needs no SDK upgrade — an unrecognized error that says `retryable: false` is still not retried.
|
|
262
|
+
|
|
263
|
+
When a response carries `Retry-After`, the next attempt waits at least that long — re-jittered, so a batch of clients rejected together does not come back in lockstep. `timeoutMs` still bounds the whole operation, retries and waits included.
|
|
264
|
+
|
|
139
265
|
## License
|
|
140
266
|
|
|
141
267
|
[Apache 2.0](./LICENSE)
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { Schema } from 'effect';
|
|
2
|
+
export declare const APPENDIX_MARKER_PREFIX = "[appendix omitted: ";
|
|
3
|
+
export declare const AppendixMarkerLevel: Schema.Literal<["off", "compact", "detailed"]>;
|
|
4
|
+
export type AppendixMarkerLevel = typeof AppendixMarkerLevel.Type;
|
|
5
|
+
/**
|
|
6
|
+
* A body with the *contents* of fenced code removed, delimiters kept. Evidence
|
|
7
|
+
* ids inside a code example are page text, not citations — but a code block's
|
|
8
|
+
* own id rides on its opening fence, so the delimiters have to stay.
|
|
9
|
+
*/
|
|
10
|
+
export declare function withoutFencedCode(markdown: string): string;
|
|
11
|
+
export interface AppendixMarker {
|
|
12
|
+
/** Evidence id of the first omitted block; on the trailing fold marker, the first block of the first folded run. */
|
|
13
|
+
readonly evidenceId?: number;
|
|
14
|
+
readonly links: number;
|
|
15
|
+
readonly approxTokens: number;
|
|
16
|
+
readonly label?: string;
|
|
17
|
+
readonly preview?: string;
|
|
18
|
+
/** On the trailing fold marker: how many further omitted runs it stands for. */
|
|
19
|
+
readonly moreBlocks?: number;
|
|
20
|
+
}
|
|
21
|
+
export declare function isAppendixMarkerLine(line: string): boolean;
|
|
22
|
+
export declare function parseAppendixMarkerLine(line: string): AppendixMarker | undefined;
|
|
23
|
+
/**
|
|
24
|
+
* Remove marker paragraphs from a Markdown body. Markers are synthetic: they
|
|
25
|
+
* are never search snippets, never citable, and never scored by quality evals.
|
|
26
|
+
*/
|
|
27
|
+
export declare function stripAppendixMarkers(markdown: string): string;
|
|
28
|
+
/** Every marker in a Markdown body, in document order. */
|
|
29
|
+
export declare function parseAppendixMarkers(markdown: string): AppendixMarker[];
|
|
30
|
+
//# sourceMappingURL=AppendixMarkers.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AppendixMarkers.d.ts","sourceRoot":"","sources":["../../src/AppendixMarkers.ts"],"names":[],"mappings":"AAcA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAA;AAE/B,eAAO,MAAM,sBAAsB,wBAAwB,CAAA;AAE3D,eAAO,MAAM,mBAAmB,gDAA+C,CAAA;AAC/E,MAAM,MAAM,mBAAmB,GAAG,OAAO,mBAAmB,CAAC,IAAI,CAAA;AAwRjE;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAM1D;AAED,MAAM,WAAW,cAAc;IAC9B,oHAAoH;IACpH,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;IACzB,gFAAgF;IAChF,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAG1D;AAED,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAgBhF;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAU7D;AAED,0DAA0D;AAC1D,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,GAAG,cAAc,EAAE,CAMvE"}
|
package/build/dts/Client.d.ts
CHANGED
|
@@ -1,7 +1,27 @@
|
|
|
1
|
-
import { type
|
|
2
|
-
|
|
1
|
+
import { type CrawlProjectionOptions, type CrawlsClient, makeCrawlProjectionsClient } from './Crawls.js';
|
|
2
|
+
import { type BatchedParams, type BatchedRequestOptions, type BatchedResponse, type CancelBatchedResponse, type ExpandServiceOptions, type FetchJsonParams, type FetchJsonRequestOptions, type FetchJsonResponse, type FetchParams, type FetchRequestOptions, type FetchResponse, type GetBatchedParams, type GetBatchedResponse, type RequestOptions, type SnapshotCatalog, type SnapshotCatalogParams, type SnapshotGetParams, type SnapshotHighlightsParams, type SnapshotHighlightsResponse } from './Service.js';
|
|
3
|
+
export type { BatchedParams, BatchedRequestOptions, BatchedResponse, CancelBatchedResponse, FetchJsonParams, FetchJsonRequestOptions, FetchJsonResponse, FetchParams, FetchRequestOptions, FetchResponse, GetBatchedParams, GetBatchedResponse, SnapshotCatalog, SnapshotCatalogParams, SnapshotGetParams, SnapshotHighlightsParams, SnapshotHighlightsResponse, } from './Service.js';
|
|
3
4
|
/** Options for constructing an {@link ExpandClient}. Same shape as `ExpandServiceOptions`. */
|
|
4
5
|
export type ExpandClientOptions = ExpandServiceOptions;
|
|
6
|
+
export declare class SnapshotService {
|
|
7
|
+
private readonly baseUrl;
|
|
8
|
+
private readonly readSnapshot;
|
|
9
|
+
constructor(baseUrl: string, readSnapshot: (params: SnapshotGetParams, options?: RequestOptions) => Promise<SnapshotCatalog | SnapshotHighlightsResponse>);
|
|
10
|
+
/** Read the public catalog, or request authenticated, billed Highlights from this Snapshot. */
|
|
11
|
+
get(params: SnapshotHighlightsParams, options?: RequestOptions): Promise<SnapshotHighlightsResponse>;
|
|
12
|
+
get(params: SnapshotCatalogParams, options?: RequestOptions): Promise<SnapshotCatalog>;
|
|
13
|
+
get(params: SnapshotGetParams, options?: RequestOptions): Promise<SnapshotCatalog | SnapshotHighlightsResponse>;
|
|
14
|
+
network(params: {
|
|
15
|
+
readonly snapshotId: string;
|
|
16
|
+
readonly q?: string;
|
|
17
|
+
}): Promise<unknown>;
|
|
18
|
+
preview(params: {
|
|
19
|
+
readonly snapshotId: string;
|
|
20
|
+
readonly artifactId: string;
|
|
21
|
+
readonly maxBytes?: number;
|
|
22
|
+
}): Promise<unknown>;
|
|
23
|
+
private request;
|
|
24
|
+
}
|
|
5
25
|
/**
|
|
6
26
|
* Promise-based client for the Expand API. Holds an {@link InternalClient} layer and runs Effects
|
|
7
27
|
* against it, translating typed failures into the `ExpandClient*Error` hierarchy that async/await
|
|
@@ -15,7 +35,21 @@ export type ExpandClientOptions = ExpandServiceOptions;
|
|
|
15
35
|
*/
|
|
16
36
|
export declare class ExpandClient {
|
|
17
37
|
private readonly layer;
|
|
38
|
+
private readonly lifetime;
|
|
39
|
+
private readonly activeRequests;
|
|
40
|
+
private dispatcher?;
|
|
41
|
+
private closePromise?;
|
|
42
|
+
/** Create, list and control crawls, and read their event streams. */
|
|
43
|
+
readonly crawls: CrawlsClient;
|
|
44
|
+
/** Read stored crawl captures: Markdown, HTML, State JSON and evidence. */
|
|
45
|
+
readonly captures: ReturnType<typeof makeCrawlProjectionsClient>['captures'];
|
|
46
|
+
/** Read a crawled page's versions and diffs. */
|
|
47
|
+
readonly pages: ReturnType<typeof makeCrawlProjectionsClient>['pages'];
|
|
48
|
+
readonly snapshots: SnapshotService;
|
|
49
|
+
private readonly projections;
|
|
18
50
|
constructor(options?: ExpandClientOptions);
|
|
51
|
+
/** Abort active requests and release this client's HTTP connections. Repeated calls are safe. */
|
|
52
|
+
close(): Promise<void>;
|
|
19
53
|
/**
|
|
20
54
|
* Fetch a URL through the Expand API.
|
|
21
55
|
*
|
|
@@ -29,12 +63,10 @@ export declare class ExpandClient {
|
|
|
29
63
|
* Fetch a URL through the object-mode endpoint.
|
|
30
64
|
*
|
|
31
65
|
* Returns `{ meta, markdown, json }`; use {@link fetch} for the raw markdown document.
|
|
66
|
+
* A string argument instead reads an existing JSON projection from this API without starting a capture.
|
|
32
67
|
*/
|
|
33
68
|
fetchJson(params: FetchJsonParams, options?: FetchJsonRequestOptions): Promise<FetchJsonResponse>;
|
|
34
|
-
|
|
35
|
-
* Search a previous Fetch response by `snapshotId` without starting a new capture.
|
|
36
|
-
*/
|
|
37
|
-
fetchSearch(params: FetchSearchParams, options?: RequestOptions): Promise<FetchSearchResponse>;
|
|
69
|
+
fetchJson(url: string, options?: CrawlProjectionOptions): Promise<unknown>;
|
|
38
70
|
/**
|
|
39
71
|
* Kick off an asynchronous batched fetch over one or more URLs. Returns a batched-run ID that can
|
|
40
72
|
* be polled via {@link getBatched}.
|
|
@@ -64,4 +96,7 @@ export declare class ExpandClient {
|
|
|
64
96
|
cancelBatched(id: string, options?: RequestOptions): Promise<CancelBatchedResponse>;
|
|
65
97
|
private run;
|
|
66
98
|
}
|
|
99
|
+
/** Concise public name used by the Crawl API examples. */
|
|
100
|
+
export declare class Expand extends ExpandClient {
|
|
101
|
+
}
|
|
67
102
|
//# sourceMappingURL=Client.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Client.d.ts","sourceRoot":"","sources":["../../src/Client.ts"],"names":[],"mappings":"AAEA,OAAO,EACN,KAAK,aAAa,EAClB,KAAK,qBAAqB,EAC1B,KAAK,eAAe,EAEpB,KAAK,qBAAqB,EAC1B,KAAK,oBAAoB,EACzB,KAAK,eAAe,EACpB,KAAK,uBAAuB,EAC5B,KAAK,iBAAiB,EACtB,KAAK,WAAW,EAChB,KAAK,mBAAmB,EACxB,KAAK,aAAa,EAClB,KAAK,
|
|
1
|
+
{"version":3,"file":"Client.d.ts","sourceRoot":"","sources":["../../src/Client.ts"],"names":[],"mappings":"AAEA,OAAO,EACN,KAAK,sBAAsB,EAC3B,KAAK,YAAY,EACjB,0BAA0B,EAE1B,MAAM,aAAa,CAAA;AAGpB,OAAO,EACN,KAAK,aAAa,EAClB,KAAK,qBAAqB,EAC1B,KAAK,eAAe,EAEpB,KAAK,qBAAqB,EAC1B,KAAK,oBAAoB,EACzB,KAAK,eAAe,EACpB,KAAK,uBAAuB,EAC5B,KAAK,iBAAiB,EACtB,KAAK,WAAW,EAChB,KAAK,mBAAmB,EACxB,KAAK,aAAa,EAClB,KAAK,gBAAgB,EACrB,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,qBAAqB,EAC1B,KAAK,iBAAiB,EACtB,KAAK,wBAAwB,EAC7B,KAAK,0BAA0B,EAC/B,MAAM,cAAc,CAAA;AAErB,YAAY,EACX,aAAa,EACb,qBAAqB,EACrB,eAAe,EACf,qBAAqB,EACrB,eAAe,EACf,uBAAuB,EACvB,iBAAiB,EACjB,WAAW,EACX,mBAAmB,EACnB,aAAa,EACb,gBAAgB,EAChB,kBAAkB,EAClB,eAAe,EACf,qBAAqB,EACrB,iBAAiB,EACjB,wBAAwB,EACxB,0BAA0B,GAC1B,MAAM,cAAc,CAAA;AAErB,8FAA8F;AAC9F,MAAM,MAAM,mBAAmB,GAAG,oBAAoB,CAAA;AAEtD,qBAAa,eAAe;IAE1B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAF9B,YACkB,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,CAC9B,MAAM,EAAE,iBAAiB,EACzB,OAAO,CAAC,EAAE,cAAc,KACpB,OAAO,CAAC,eAAe,GAAG,0BAA0B,CAAC,EACvD;IAEJ,+FAA+F;IAC/F,GAAG,CAAC,MAAM,EAAE,wBAAwB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,0BAA0B,CAAC,CAAA;IACpG,GAAG,CAAC,MAAM,EAAE,qBAAqB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,eAAe,CAAC,CAAA;IACtF,GAAG,CAAC,MAAM,EAAE,iBAAiB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,eAAe,GAAG,0BAA0B,CAAC,CAAA;IAKzG,OAAO,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAE5F;IAEK,OAAO,CAAC,MAAM,EAAE;QACrB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;QAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;QAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,CAGnB;YAEa,OAAO;CAKrB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,YAAY;IACxB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA6B;IACnD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAwB;IACjD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA8B;IAC7D,OAAO,CAAC,UAAU,CAAC,CAA0B;IAC7C,OAAO,CAAC,YAAY,CAAC,CAAe;IACpC,qEAAqE;IACrE,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAA;IAC7B,2EAA2E;IAC3E,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC,OAAO,0BAA0B,CAAC,CAAC,UAAU,CAAC,CAAA;IAC5E,gDAAgD;IAChD,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC,OAAO,0BAA0B,CAAC,CAAC,OAAO,CAAC,CAAA;IACtE,QAAQ,CAAC,SAAS,EAAE,eAAe,CAAA;IACnC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA+C;IAE3E,YAAY,OAAO,GAAE,mBAAwB,EA2C5C;IAED,iGAAiG;IACjG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAMrB;IAED;;;;;;;OAOG;IACG,KAAK,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,aAAa,CAAC,CAE1F;IAED;;;;;OAKG;IACH,SAAS,CAAC,MAAM,EAAE,eAAe,EAAE,OAAO,CAAC,EAAE,uBAAuB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAA;IACjG,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;IAS1E;;;;;;;;;;;;OAYG;IACG,OAAO,CAAC,MAAM,EAAE,aAAa,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,eAAe,CAAC,CAElG;IAED;;;;;;;OAOG;IACG,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAgB,GAAG,cAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAEzG;IAED;;OAEG;IACG,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAE5F;YAEa,GAAG;CAejB;AAED,0DAA0D;AAC1D,qBAAa,MAAO,SAAQ,YAAY;CAAG"}
|