@expandai/sdk 0.19.0 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/README.md +130 -4
  2. package/build/dts/AppendixMarkers.d.ts +30 -0
  3. package/build/dts/AppendixMarkers.d.ts.map +1 -0
  4. package/build/dts/Client.d.ts +41 -6
  5. package/build/dts/Client.d.ts.map +1 -1
  6. package/build/dts/Crawls.d.ts +568 -0
  7. package/build/dts/Crawls.d.ts.map +1 -0
  8. package/build/dts/EnvProxyHttpClient.d.ts +28 -0
  9. package/build/dts/EnvProxyHttpClient.d.ts.map +1 -0
  10. package/build/dts/FetchJson.d.ts +444 -37
  11. package/build/dts/FetchJson.d.ts.map +1 -1
  12. package/build/dts/Generated.d.ts +1545 -788
  13. package/build/dts/Generated.d.ts.map +1 -1
  14. package/build/dts/HttpClientHelpers.d.ts +13 -0
  15. package/build/dts/HttpClientHelpers.d.ts.map +1 -1
  16. package/build/dts/Internal.d.ts +6 -4
  17. package/build/dts/Internal.d.ts.map +1 -1
  18. package/build/dts/ParseErrorBody.d.ts +26 -0
  19. package/build/dts/ParseErrorBody.d.ts.map +1 -0
  20. package/build/dts/Playground.d.ts +20 -0
  21. package/build/dts/Playground.d.ts.map +1 -1
  22. package/build/dts/RetryAfter.d.ts +40 -0
  23. package/build/dts/RetryAfter.d.ts.map +1 -0
  24. package/build/dts/Service.d.ts +94 -45
  25. package/build/dts/Service.d.ts.map +1 -1
  26. package/build/dts/StreamPage.d.ts +10 -0
  27. package/build/dts/StreamPage.d.ts.map +1 -0
  28. package/build/dts/index.d.ts +9 -2
  29. package/build/dts/index.d.ts.map +1 -1
  30. package/build/esm/AppendixMarkers.js +307 -0
  31. package/build/esm/AppendixMarkers.js.map +1 -0
  32. package/build/esm/Client.js +96 -15
  33. package/build/esm/Client.js.map +1 -1
  34. package/build/esm/Crawls.js +786 -0
  35. package/build/esm/Crawls.js.map +1 -0
  36. package/build/esm/EnvProxyHttpClient.js +55 -0
  37. package/build/esm/EnvProxyHttpClient.js.map +1 -0
  38. package/build/esm/FetchJson.js +48 -22
  39. package/build/esm/FetchJson.js.map +1 -1
  40. package/build/esm/Generated.js +540 -112
  41. package/build/esm/Generated.js.map +1 -1
  42. package/build/esm/HttpClientHelpers.js +36 -1
  43. package/build/esm/HttpClientHelpers.js.map +1 -1
  44. package/build/esm/Internal.js +140 -33
  45. package/build/esm/Internal.js.map +1 -1
  46. package/build/esm/ParseErrorBody.js +41 -0
  47. package/build/esm/ParseErrorBody.js.map +1 -0
  48. package/build/esm/Playground.js +38 -0
  49. package/build/esm/Playground.js.map +1 -1
  50. package/build/esm/RetryAfter.js +104 -0
  51. package/build/esm/RetryAfter.js.map +1 -0
  52. package/build/esm/Service.js +96 -17
  53. package/build/esm/Service.js.map +1 -1
  54. package/build/esm/StreamPage.js +46 -0
  55. package/build/esm/StreamPage.js.map +1 -0
  56. package/build/esm/index.js +5 -1
  57. package/build/esm/index.js.map +1 -1
  58. package/package.json +8 -2
  59. package/src/AppendixMarkers.ts +370 -0
  60. package/src/Client.ts +131 -16
  61. package/src/Crawls.ts +1387 -0
  62. package/src/EnvProxyHttpClient.ts +71 -0
  63. package/src/FetchJson.ts +63 -25
  64. package/src/Generated.ts +738 -162
  65. package/src/HttpClientHelpers.ts +48 -1
  66. package/src/Internal.ts +163 -105
  67. package/src/ParseErrorBody.ts +42 -0
  68. package/src/Playground.ts +48 -0
  69. package/src/RetryAfter.ts +138 -0
  70. package/src/Service.ts +208 -25
  71. package/src/StreamPage.ts +47 -0
  72. package/src/index.ts +29 -3
package/README.md CHANGED
@@ -6,6 +6,8 @@ Built on [Effect](https://effect.website), with both a class-based client for pr
6
6
 
7
7
  ## Install
8
8
 
9
+ Requires Node.js 20.3 or newer.
10
+
9
11
  ```sh
10
12
  npm install @expandai/sdk
11
13
  # or
@@ -14,6 +16,14 @@ bun add @expandai/sdk
14
16
  pnpm add @expandai/sdk
15
17
  ```
16
18
 
19
+ Internal staging builds are available without changing the production `latest` tag:
20
+
21
+ ```sh
22
+ npm install @expandai/sdk@staging
23
+ ```
24
+
25
+ Pass the staging API URL through the client's `baseUrl`; selecting the npm tag does not change the default production endpoint.
26
+
17
27
  ## Authentication
18
28
 
19
29
  Set `EXPAND_API_KEY` in your environment, or pass `apiKey` explicitly when constructing the client.
@@ -31,10 +41,16 @@ import { ExpandClient } from '@expandai/sdk'
31
41
 
32
42
  const expand = new ExpandClient()
33
43
 
34
- const result = await expand.fetch({ url: 'https://example.com' })
35
- console.log(result)
44
+ try {
45
+ const result = await expand.fetch({ url: 'https://example.com' })
46
+ console.log(result)
47
+ } finally {
48
+ await expand.close()
49
+ }
36
50
  ```
37
51
 
52
+ Reuse a client for the lifetime of your application, then call `close()` during shutdown. Closing aborts active requests, ends Crawl iterators and polling, and releases this client's connections without closing another client's pool. Repeated closes are safe; later requests on a closed client reject. Effect service consumers retain the existing scoped-layer cleanup.
53
+
38
54
  Constructor options:
39
55
 
40
56
  ```ts
@@ -42,7 +58,7 @@ new ExpandClient({
42
58
  apiKey: 'xpnd_...', // defaults to process.env.EXPAND_API_KEY
43
59
  baseUrl: 'https://api...', // defaults to https://api.expand.ai
44
60
  timeoutMs: 60_000, // total wall-clock budget incl. retries
45
- maxRetries: 2, // retries on timeout / connection / 408,409,429,5xx
61
+ maxRetries: 2, // retries transient failures; see "What gets retried"
46
62
  })
47
63
  ```
48
64
 
@@ -76,7 +92,7 @@ ExpandService.layer({
76
92
  apiKey: 'xpnd_...', // defaults to process.env.EXPAND_API_KEY
77
93
  baseUrl: 'https://api...', // defaults to https://api.expand.ai
78
94
  timeoutMs: 60_000, // total wall-clock budget incl. retries
79
- maxRetries: 2, // retries on timeout / connection / 408,409,429,5xx
95
+ maxRetries: 2, // retries transient failures; see "What gets retried"
80
96
  })
81
97
  ```
82
98
 
@@ -87,6 +103,95 @@ Typed errors in the `Effect` channel:
87
103
  - `ExpandConnectionError` — network failure
88
104
  - `ExpandTimeoutError` — request timed out
89
105
 
106
+ ## Crawl and capture projections
107
+
108
+ Creation and run-command replays require a caller-persisted idempotency key. One-off crawls close permanently when done, failed, or canceled; create a new crawl to run again. Events carry their own resume offset.
109
+
110
+ Replaying a creation key returns the original ordered Crawl IDs, even if a member has since expired. Reading an expired member returns 404; replay never recreates it or starts a replacement run.
111
+
112
+ ### How a crawl finds pages
113
+
114
+ A crawl starts at each source URL and stays on that URL's registrable domain and its subdomains.
115
+
116
+ - **`maxDepth` counts clicks from your URL.** `0` is the URL alone. `1` adds the pages it links to and the pages in the site's sitemap. `2` adds the pages those link to, and so on. Pages found in the sitemap or by search count as one click, because they were not on your list. A `kind: 'sitemap'` source is the exception: the URLs it lists are yours, so they start at depth 0. The default, 50, is the whole site up to `maxPages`.
117
+ - **`include` decides what ends up in the results, not where the crawler goes.** To reach `/p/**` from the homepage, the crawler opens the pages that link there, such as `/` and `/damen`, and follows their links. It does not keep those pages, and they do not count toward `maxPages`.
118
+ - **`exclude` keeps the crawler out.** A matching page is never opened, so nothing behind it is found through it.
119
+ - **`sync.freshness` is how often every page is checked for changes**, for example `24h`. How each check is made is up to the crawler.
120
+
121
+ `changePolicy.ignore` removes matching subtrees from change comparison; `changePolicy.volatile` replaces them with stable markers. Both leave saved page content unchanged. Selectors support static tags, classes, IDs, attributes, combinators and structural pseudo-classes such as `:not`, `:has` and `:nth-child`. Interaction states such as `:hover`, pseudo-elements, named namespace prefixes, nested `:has` and `:nth-child(... of ...)` are rejected. Matching includes captured shadow subtrees and uses case-sensitive class and ID values even on quirks-mode pages.
122
+
123
+ ```ts
124
+ import { Expand } from '@expandai/sdk'
125
+
126
+ const expand = new Expand({ apiKey: process.env.EXPAND_API_KEY })
127
+ const [crawl] = await expand.crawls.create(
128
+ { sources: ['https://example.com/products'], formats: ['markdown', 'stateJson'] },
129
+ { idempotencyKey: 'catalog-import-2026-09' },
130
+ )
131
+ if (!crawl) throw new Error('Expected a Crawl')
132
+
133
+ for await (const event of crawl.events()) {
134
+ if (event.type !== 'page.added' && event.type !== 'page.changed') continue
135
+ const capture = expand.captures(event.captureId)
136
+ const state = event.page.stateJson ?? await capture.stateJson()
137
+ for (const evidence of state.evidence) {
138
+ const body = evidence.body !== undefined ? evidence.body : (evidence.bodyUrl ? await expand.fetchJson(evidence.bodyUrl) : undefined)
139
+ // Process body, then persist event.offset in your application.
140
+ }
141
+ }
142
+ ```
143
+
144
+ `captures(id)` also exposes `get()`, `markdown()`, `html()`, `evidence(key)` and `playgroundUrl()`. These read stored captures, not fresh pages. JSON body URLs must belong to the configured API origin and cannot redirect. Projection reads accept `signal` and `timeoutMs`; they do not perform automatic retries.
145
+
146
+ Catalog and version lists support individual items, pages, or one awaited page:
147
+
148
+ ```ts
149
+ const list = crawl.pages({ limit: 100, status: 'live' })
150
+ const first = await list // { items, nextCursor? }
151
+ for await (const page of list.pages()) { /* page.items */ }
152
+ for await (const item of list) { /* item.id, item.url */ }
153
+ const bounded = await list.collect({ limit: 200 })
154
+
155
+ const versions = expand.pages('page-id').versions({ limit: 20 })
156
+ const diff = await expand.pages('page-id').diff({ from: 'capture-before', to: 'capture-after' })
157
+ ```
158
+
159
+ A list instance caches its first page. Stop iteration with `break`, or cancel reads through `signal`. `crawl.page(id)` and `crawl.pageByUrl(url)` read the current catalog projection. Page-version lists use `before` or `at` for historical reads.
160
+
161
+ `events({ from: '-1' })` replays retained history, including changes, removals, and failures. It does not rebuild the current catalog. For a full catalog import or recovery after `CrawlOffsetExpiredError`, use the pinned snapshot:
162
+
163
+ ```ts
164
+ for await (const batch of crawl.snapshot().pages()) {
165
+ // Apply batch.items, then persist batch.nextOffset.
166
+ // When batch.upToDate is true, that offset is the boundary to resume with crawl.events({ from }).
167
+ }
168
+ ```
169
+
170
+ `snapshot({ from: savedSnapshotCursor })` resumes an unfinished snapshot. Snapshot cursors belong to `/snapshot`; an older bootstrap checkpoint submitted to `/events` returns 410. Event offsets are opaque and sortable: copy them exactly rather than converting them to numbers. One-off event and activity iterators stop at stream EOF. Recurring reads return when caught up unless `live: true` is set.
171
+
172
+ Event reads also expose bounded server pages. Awaiting an event read returns one page, including an empty caught-up page, and closes that read. Resume using its `nextOffset`. Event reads do not cache that first page across separate invocations.
173
+
174
+ Each non-live event request and control request honors the client's `timeoutMs` across its retries and `maxRetries` limit. Explicit `live: true` reads reconnect indefinitely on transient failures until canceled, returned from, or closed with the client. A server refusal marked `retryable: false` stops either mode. `done()` polls until completion or cancellation; its finite `pollIntervalMs` must be no greater than 2147483647, and values below 50 are clamped to 50ms.
175
+
176
+ Crawl API refusals extend `ExpandClientApiError` and `ExpandClientError`, with `status`, `body`, `code`, and `retryable` available. `CrawlOffsetExpiredError` remains the specific 410 error. Local validation errors extend `ExpandClientError` through `CrawlClientError`.
177
+
178
+ Effect users can call `yield* expand.crawls.create(request, { idempotencyKey })` through `ExpandService`, then use `status`, `update`, `cancel`, or `run` on the returned resources. These control methods share the service's HTTP client, retries, and typed `ExpandError` channel. Event iterators, `done()`, and capture projections currently use the Promise interface; the Effect service does not yet mirror those helpers.
179
+
180
+ Because these reads are thenable, `Promise.resolve`, `Promise.all`, or returning one from an `async` function also starts that one-page read. Live page iteration exposes empty long-poll timeout pages; item iteration hides them.
181
+
182
+ ```ts
183
+ const batch = await crawl.events({ from: savedOffset })
184
+ if (batch) {
185
+ for (const event of batch.items) { /* process event */ }
186
+ // Persist batch.nextOffset after processing the whole batch.
187
+ }
188
+ for await (const batch of crawl.events({ live: true }).pages()) {
189
+ // batch.crawlId identifies the partition, including in merged reads.
190
+ }
191
+ ```
192
+
193
+ Merged page reads take `from: { [crawlId]: offset }`, just like merged item reads. Each returned batch belongs to one Crawl and carries that Crawl's `nextOffset`; never reuse it for a different Crawl.
194
+
90
195
  ## Batched fetches
91
196
 
92
197
  `batched` kicks off a job and returns immediately with an `id`. Poll `getBatched(id)` until `batchedStatus` is no longer `RUNNING` or `QUEUED`.
@@ -109,6 +214,8 @@ while (status.batchedStatus === 'RUNNING' || status.batchedStatus === 'QUEUED')
109
214
  console.log(status.results)
110
215
  ```
111
216
 
217
+ Successful results include Markdown at `data.markdown` and State JSON at `data.json` by default when extraction succeeds.
218
+
112
219
  Each result has a persisted `status` (`QUEUED`, `RUNNING`, `SUCCEEDED`, `FAILED`, or `CANCELLED`) alongside its `data`.
113
220
 
114
221
  ### Effect service
@@ -136,6 +243,25 @@ NodeRuntime.runMain(program.pipe(Effect.provide(ExpandService.layer({ apiKey: pr
136
243
 
137
244
  The SDK generates an idempotency key and reuses it for automatic retries when one is omitted. Supply a stable `idempotencyKey` to deduplicate separate calls or calls made after a process restart. Reusing a key with a different payload returns `409 BatchedIdempotencyConflict`.
138
245
 
246
+ ## What gets retried
247
+
248
+ Not every 5xx is transient. Some of them are verdicts about the page you asked for — the browser committed its own error document, or the page could not be captured after the capture retries were spent — and repeating the request re-runs a full capture to reach the same answer, on your timeout budget and your quota.
249
+
250
+ So the server states the answer rather than leaving it to a status-code rule. Error bodies whose status is ambiguous (`429`, `500`, `502`, `503`, `504`, `529`) carry a `retryable` boolean, and the SDK follows it:
251
+
252
+ | Response | Retried |
253
+ | --- | --- |
254
+ | Timeout, connection failure | yes |
255
+ | `retryable: true` (e.g. `429 TooManyRequests`, `503 ServiceUnavailable`, `529 FetchCapacityTimeout`) | yes |
256
+ | `retryable: false` (e.g. `502 FetchNavigationFailed`, `502 FetchCaptureFailed`, `504 FetchCaptureTimeout`) | no |
257
+ | A verdict tag with no `retryable` field — `409 BatchedIdempotencyConflict`, `409 BatchedCancellationConflict`, `502 FetchNavigationFailed`, `502 FetchCaptureFailed`, `504 FetchCaptureTimeout` | no |
258
+ | `408`, `429`, other `5xx` with no `retryable` field | yes |
259
+ | Any other non-2xx | no |
260
+
261
+ Because the verdict travels in the body, a newly-terminal error class needs no SDK upgrade — an unrecognized error that says `retryable: false` is still not retried.
262
+
263
+ When a response carries `Retry-After`, the next attempt waits at least that long — re-jittered, so a batch of clients rejected together does not come back in lockstep. `timeoutMs` still bounds the whole operation, retries and waits included.
264
+
139
265
  ## License
140
266
 
141
267
  [Apache 2.0](./LICENSE)
@@ -0,0 +1,30 @@
1
+ import { Schema } from 'effect';
2
+ export declare const APPENDIX_MARKER_PREFIX = "[appendix omitted: ";
3
+ export declare const AppendixMarkerLevel: Schema.Literal<["off", "compact", "detailed"]>;
4
+ export type AppendixMarkerLevel = typeof AppendixMarkerLevel.Type;
5
+ /**
6
+ * A body with the *contents* of fenced code removed, delimiters kept. Evidence
7
+ * ids inside a code example are page text, not citations — but a code block's
8
+ * own id rides on its opening fence, so the delimiters have to stay.
9
+ */
10
+ export declare function withoutFencedCode(markdown: string): string;
11
+ export interface AppendixMarker {
12
+ /** Evidence id of the first omitted block; on the trailing fold marker, the first block of the first folded run. */
13
+ readonly evidenceId?: number;
14
+ readonly links: number;
15
+ readonly approxTokens: number;
16
+ readonly label?: string;
17
+ readonly preview?: string;
18
+ /** On the trailing fold marker: how many further omitted runs it stands for. */
19
+ readonly moreBlocks?: number;
20
+ }
21
+ export declare function isAppendixMarkerLine(line: string): boolean;
22
+ export declare function parseAppendixMarkerLine(line: string): AppendixMarker | undefined;
23
+ /**
24
+ * Remove marker paragraphs from a Markdown body. Markers are synthetic: they
25
+ * are never search snippets, never citable, and never scored by quality evals.
26
+ */
27
+ export declare function stripAppendixMarkers(markdown: string): string;
28
+ /** Every marker in a Markdown body, in document order. */
29
+ export declare function parseAppendixMarkers(markdown: string): AppendixMarker[];
30
+ //# sourceMappingURL=AppendixMarkers.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"AppendixMarkers.d.ts","sourceRoot":"","sources":["../../src/AppendixMarkers.ts"],"names":[],"mappings":"AAcA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAA;AAE/B,eAAO,MAAM,sBAAsB,wBAAwB,CAAA;AAE3D,eAAO,MAAM,mBAAmB,gDAA+C,CAAA;AAC/E,MAAM,MAAM,mBAAmB,GAAG,OAAO,mBAAmB,CAAC,IAAI,CAAA;AAwRjE;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAM1D;AAED,MAAM,WAAW,cAAc;IAC9B,oHAAoH;IACpH,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;IACzB,gFAAgF;IAChF,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAG1D;AAED,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAgBhF;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAU7D;AAED,0DAA0D;AAC1D,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,GAAG,cAAc,EAAE,CAMvE"}
@@ -1,7 +1,27 @@
1
- import { type BatchedParams, type BatchedRequestOptions, type BatchedResponse, type CancelBatchedResponse, type ExpandServiceOptions, type FetchJsonParams, type FetchJsonRequestOptions, type FetchJsonResponse, type FetchParams, type FetchRequestOptions, type FetchResponse, type FetchSearchParams, type FetchSearchResponse, type GetBatchedParams, type GetBatchedResponse, type RequestOptions } from './Service.js';
2
- export type { BatchedParams, BatchedRequestOptions, BatchedResponse, CancelBatchedResponse, FetchJsonParams, FetchJsonRequestOptions, FetchJsonResponse, FetchParams, FetchRequestOptions, FetchResponse, FetchSearchParams, FetchSearchResponse, GetBatchedParams, GetBatchedResponse, } from './Service.js';
1
+ import { type CrawlProjectionOptions, type CrawlsClient, makeCrawlProjectionsClient } from './Crawls.js';
2
+ import { type BatchedParams, type BatchedRequestOptions, type BatchedResponse, type CancelBatchedResponse, type ExpandServiceOptions, type FetchJsonParams, type FetchJsonRequestOptions, type FetchJsonResponse, type FetchParams, type FetchRequestOptions, type FetchResponse, type GetBatchedParams, type GetBatchedResponse, type RequestOptions, type SnapshotCatalog, type SnapshotCatalogParams, type SnapshotGetParams, type SnapshotHighlightsParams, type SnapshotHighlightsResponse } from './Service.js';
3
+ export type { BatchedParams, BatchedRequestOptions, BatchedResponse, CancelBatchedResponse, FetchJsonParams, FetchJsonRequestOptions, FetchJsonResponse, FetchParams, FetchRequestOptions, FetchResponse, GetBatchedParams, GetBatchedResponse, SnapshotCatalog, SnapshotCatalogParams, SnapshotGetParams, SnapshotHighlightsParams, SnapshotHighlightsResponse, } from './Service.js';
3
4
  /** Options for constructing an {@link ExpandClient}. Same shape as `ExpandServiceOptions`. */
4
5
  export type ExpandClientOptions = ExpandServiceOptions;
6
+ export declare class SnapshotService {
7
+ private readonly baseUrl;
8
+ private readonly readSnapshot;
9
+ constructor(baseUrl: string, readSnapshot: (params: SnapshotGetParams, options?: RequestOptions) => Promise<SnapshotCatalog | SnapshotHighlightsResponse>);
10
+ /** Read the public catalog, or request authenticated, billed Highlights from this Snapshot. */
11
+ get(params: SnapshotHighlightsParams, options?: RequestOptions): Promise<SnapshotHighlightsResponse>;
12
+ get(params: SnapshotCatalogParams, options?: RequestOptions): Promise<SnapshotCatalog>;
13
+ get(params: SnapshotGetParams, options?: RequestOptions): Promise<SnapshotCatalog | SnapshotHighlightsResponse>;
14
+ network(params: {
15
+ readonly snapshotId: string;
16
+ readonly q?: string;
17
+ }): Promise<unknown>;
18
+ preview(params: {
19
+ readonly snapshotId: string;
20
+ readonly artifactId: string;
21
+ readonly maxBytes?: number;
22
+ }): Promise<unknown>;
23
+ private request;
24
+ }
5
25
  /**
6
26
  * Promise-based client for the Expand API. Holds an {@link InternalClient} layer and runs Effects
7
27
  * against it, translating typed failures into the `ExpandClient*Error` hierarchy that async/await
@@ -15,7 +35,21 @@ export type ExpandClientOptions = ExpandServiceOptions;
15
35
  */
16
36
  export declare class ExpandClient {
17
37
  private readonly layer;
38
+ private readonly lifetime;
39
+ private readonly activeRequests;
40
+ private dispatcher?;
41
+ private closePromise?;
42
+ /** Create, list and control crawls, and read their event streams. */
43
+ readonly crawls: CrawlsClient;
44
+ /** Read stored crawl captures: Markdown, HTML, State JSON and evidence. */
45
+ readonly captures: ReturnType<typeof makeCrawlProjectionsClient>['captures'];
46
+ /** Read a crawled page's versions and diffs. */
47
+ readonly pages: ReturnType<typeof makeCrawlProjectionsClient>['pages'];
48
+ readonly snapshots: SnapshotService;
49
+ private readonly projections;
18
50
  constructor(options?: ExpandClientOptions);
51
+ /** Abort active requests and release this client's HTTP connections. Repeated calls are safe. */
52
+ close(): Promise<void>;
19
53
  /**
20
54
  * Fetch a URL through the Expand API.
21
55
  *
@@ -29,12 +63,10 @@ export declare class ExpandClient {
29
63
  * Fetch a URL through the object-mode endpoint.
30
64
  *
31
65
  * Returns `{ meta, markdown, json }`; use {@link fetch} for the raw markdown document.
66
+ * A string argument instead reads an existing JSON projection from this API without starting a capture.
32
67
  */
33
68
  fetchJson(params: FetchJsonParams, options?: FetchJsonRequestOptions): Promise<FetchJsonResponse>;
34
- /**
35
- * Search a previous Fetch response by `snapshotId` without starting a new capture.
36
- */
37
- fetchSearch(params: FetchSearchParams, options?: RequestOptions): Promise<FetchSearchResponse>;
69
+ fetchJson(url: string, options?: CrawlProjectionOptions): Promise<unknown>;
38
70
  /**
39
71
  * Kick off an asynchronous batched fetch over one or more URLs. Returns a batched-run ID that can
40
72
  * be polled via {@link getBatched}.
@@ -64,4 +96,7 @@ export declare class ExpandClient {
64
96
  cancelBatched(id: string, options?: RequestOptions): Promise<CancelBatchedResponse>;
65
97
  private run;
66
98
  }
99
+ /** Concise public name used by the Crawl API examples. */
100
+ export declare class Expand extends ExpandClient {
101
+ }
67
102
  //# sourceMappingURL=Client.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"Client.d.ts","sourceRoot":"","sources":["../../src/Client.ts"],"names":[],"mappings":"AAEA,OAAO,EACN,KAAK,aAAa,EAClB,KAAK,qBAAqB,EAC1B,KAAK,eAAe,EAEpB,KAAK,qBAAqB,EAC1B,KAAK,oBAAoB,EACzB,KAAK,eAAe,EACpB,KAAK,uBAAuB,EAC5B,KAAK,iBAAiB,EACtB,KAAK,WAAW,EAChB,KAAK,mBAAmB,EACxB,KAAK,aAAa,EAClB,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACxB,KAAK,gBAAgB,EACrB,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,MAAM,cAAc,CAAA;AAErB,YAAY,EACX,aAAa,EACb,qBAAqB,EACrB,eAAe,EACf,qBAAqB,EACrB,eAAe,EACf,uBAAuB,EACvB,iBAAiB,EACjB,WAAW,EACX,mBAAmB,EACnB,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,gBAAgB,EAChB,kBAAkB,GAClB,MAAM,cAAc,CAAA;AAErB,8FAA8F;AAC9F,MAAM,MAAM,mBAAmB,GAAG,oBAAoB,CAAA;AAEtD;;;;;;;;;;GAUG;AACH,qBAAa,YAAY;IACxB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA6B;IAEnD,YAAY,OAAO,GAAE,mBAAwB,EAW5C;IAED;;;;;;;OAOG;IACG,KAAK,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,aAAa,CAAC,CAE1F;IAED;;;;OAIG;IACG,SAAS,CAAC,MAAM,EAAE,eAAe,EAAE,OAAO,GAAE,uBAA4B,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAE1G;IAED;;OAEG;IACG,WAAW,CAAC,MAAM,EAAE,iBAAiB,EAAE,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAEvG;IAED;;;;;;;;;;;;OAYG;IACG,OAAO,CAAC,MAAM,EAAE,aAAa,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,eAAe,CAAC,CAElG;IAED;;;;;;;OAOG;IACG,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAgB,GAAG,cAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAEzG;IAED;;OAEG;IACG,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAE5F;YAEa,GAAG;CAOjB"}
1
+ {"version":3,"file":"Client.d.ts","sourceRoot":"","sources":["../../src/Client.ts"],"names":[],"mappings":"AAEA,OAAO,EACN,KAAK,sBAAsB,EAC3B,KAAK,YAAY,EACjB,0BAA0B,EAE1B,MAAM,aAAa,CAAA;AAGpB,OAAO,EACN,KAAK,aAAa,EAClB,KAAK,qBAAqB,EAC1B,KAAK,eAAe,EAEpB,KAAK,qBAAqB,EAC1B,KAAK,oBAAoB,EACzB,KAAK,eAAe,EACpB,KAAK,uBAAuB,EAC5B,KAAK,iBAAiB,EACtB,KAAK,WAAW,EAChB,KAAK,mBAAmB,EACxB,KAAK,aAAa,EAClB,KAAK,gBAAgB,EACrB,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,qBAAqB,EAC1B,KAAK,iBAAiB,EACtB,KAAK,wBAAwB,EAC7B,KAAK,0BAA0B,EAC/B,MAAM,cAAc,CAAA;AAErB,YAAY,EACX,aAAa,EACb,qBAAqB,EACrB,eAAe,EACf,qBAAqB,EACrB,eAAe,EACf,uBAAuB,EACvB,iBAAiB,EACjB,WAAW,EACX,mBAAmB,EACnB,aAAa,EACb,gBAAgB,EAChB,kBAAkB,EAClB,eAAe,EACf,qBAAqB,EACrB,iBAAiB,EACjB,wBAAwB,EACxB,0BAA0B,GAC1B,MAAM,cAAc,CAAA;AAErB,8FAA8F;AAC9F,MAAM,MAAM,mBAAmB,GAAG,oBAAoB,CAAA;AAEtD,qBAAa,eAAe;IAE1B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAF9B,YACkB,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,CAC9B,MAAM,EAAE,iBAAiB,EACzB,OAAO,CAAC,EAAE,cAAc,KACpB,OAAO,CAAC,eAAe,GAAG,0BAA0B,CAAC,EACvD;IAEJ,+FAA+F;IAC/F,GAAG,CAAC,MAAM,EAAE,wBAAwB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,0BAA0B,CAAC,CAAA;IACpG,GAAG,CAAC,MAAM,EAAE,qBAAqB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,eAAe,CAAC,CAAA;IACtF,GAAG,CAAC,MAAM,EAAE,iBAAiB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,eAAe,GAAG,0BAA0B,CAAC,CAAA;IAKzG,OAAO,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAE5F;IAEK,OAAO,CAAC,MAAM,EAAE;QACrB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;QAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;QAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,CAGnB;YAEa,OAAO;CAKrB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,YAAY;IACxB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA6B;IACnD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAwB;IACjD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA8B;IAC7D,OAAO,CAAC,UAAU,CAAC,CAA0B;IAC7C,OAAO,CAAC,YAAY,CAAC,CAAe;IACpC,qEAAqE;IACrE,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAA;IAC7B,2EAA2E;IAC3E,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC,OAAO,0BAA0B,CAAC,CAAC,UAAU,CAAC,CAAA;IAC5E,gDAAgD;IAChD,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC,OAAO,0BAA0B,CAAC,CAAC,OAAO,CAAC,CAAA;IACtE,QAAQ,CAAC,SAAS,EAAE,eAAe,CAAA;IACnC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA+C;IAE3E,YAAY,OAAO,GAAE,mBAAwB,EA2C5C;IAED,iGAAiG;IACjG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAMrB;IAED;;;;;;;OAOG;IACG,KAAK,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,aAAa,CAAC,CAE1F;IAED;;;;;OAKG;IACH,SAAS,CAAC,MAAM,EAAE,eAAe,EAAE,OAAO,CAAC,EAAE,uBAAuB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAA;IACjG,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;IAS1E;;;;;;;;;;;;OAYG;IACG,OAAO,CAAC,MAAM,EAAE,aAAa,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,eAAe,CAAC,CAElG;IAED;;;;;;;OAOG;IACG,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAgB,GAAG,cAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAEzG;IAED;;OAEG;IACG,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAE5F;YAEa,GAAG;CAejB;AAED,0DAA0D;AAC1D,qBAAa,MAAO,SAAQ,YAAY;CAAG"}