@outcrawl/sdk 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +198 -0
- package/dist/index.js +2493 -0
- package/dist/types/_deps/core/agent-alias.d.ts +219 -0
- package/dist/types/_deps/core/brand.d.ts +62 -0
- package/dist/types/_deps/core/certificate.d.ts +100 -0
- package/dist/types/_deps/core/cron.d.ts +116 -0
- package/dist/types/_deps/core/errors.d.ts +340 -0
- package/dist/types/_deps/core/index.d.ts +17 -0
- package/dist/types/_deps/core/money.d.ts +108 -0
- package/dist/types/_deps/core/registry.d.ts +466 -0
- package/dist/types/_deps/core/rules.d.ts +237 -0
- package/dist/types/_deps/core/secrets.d.ts +278 -0
- package/dist/types/_deps/core/types.d.ts +1677 -0
- package/dist/types/_deps/integrations/connector.d.ts +153 -0
- package/dist/types/_deps/replay/events.d.ts +1004 -0
- package/dist/types/agent.d.ts +313 -0
- package/dist/types/availability.d.ts +86 -0
- package/dist/types/browser.d.ts +107 -0
- package/dist/types/client.d.ts +128 -0
- package/dist/types/hands.d.ts +259 -0
- package/dist/types/index.d.ts +111 -0
- package/dist/types/integrations.d.ts +63 -0
- package/dist/types/monitors.d.ts +24 -0
- package/dist/types/page.d.ts +96 -0
- package/dist/types/profiles.d.ts +26 -0
- package/dist/types/result.d.ts +90 -0
- package/dist/types/rules.d.ts +41 -0
- package/dist/types/scrape.d.ts +68 -0
- package/dist/types/secrets.d.ts +36 -0
- package/dist/types/sessions.d.ts +124 -0
- package/dist/types/transport.d.ts +250 -0
- package/package.json +70 -0
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Agent — a natural-language browser task under hard caps, as a JOB.
|
|
3
|
+
*
|
|
4
|
+
* `caps` is required, at the type level and at runtime. The customer sets a
|
|
5
|
+
* ceiling before spending rather than disputing a bill afterwards, and since
|
|
6
|
+
* failures cost more than successes — agents flail, retry and loop — the same
|
|
7
|
+
* cap protects our margin as much as their wallet. Published task success rates
|
|
8
|
+
* are 42–71%, so a run that ends `partial` is normal, and the schema-shaped
|
|
9
|
+
* data it did get comes back rather than being thrown away.
|
|
10
|
+
*
|
|
11
|
+
* `schema` is OPTIONAL and omitting it does not cost you the answer. With a
|
|
12
|
+
* schema, `run.data` is schema-shaped and satisfying it is the success signal.
|
|
13
|
+
* Without one it is the agent's own closing summary — a string — so a question
|
|
14
|
+
* asked in plain language is answered in plain language. A task that asks
|
|
15
|
+
* nothing, "click the unsubscribe link", returns no `data` at all; the field is
|
|
16
|
+
* optional and its absence is a success, not an error.
|
|
17
|
+
*
|
|
18
|
+
* ── What changed, and why every caller has to know ───────────────────────────
|
|
19
|
+
*
|
|
20
|
+
* `oc.agent(...)` used to BE the run: awaiting it blocked for as long as the
|
|
21
|
+
* task took and iterating it streamed steps off a held socket. Both were the
|
|
22
|
+
* same HTTP request, so both died at the platform's fifteen-minute forward
|
|
23
|
+
* timeout with no terminal frame.
|
|
24
|
+
*
|
|
25
|
+
* Submitting now returns in milliseconds with an id. The run outlives the
|
|
26
|
+
* request, hours are normal, and there is no duration ceiling. `await` still
|
|
27
|
+
* works and still gives you the finished run — see {@link AgentJob.then} — but
|
|
28
|
+
* it is a POLL over a durable row rather than a socket, so a dropped connection
|
|
29
|
+
* costs nothing and a killed browser costs nothing. `for await` still gives you
|
|
30
|
+
* the log as it happens — see {@link AgentEvents} — but it is a CURSOR TAIL
|
|
31
|
+
* over durable rows, so a consumer that dies resumes at the sequence it had
|
|
32
|
+
* reached instead of losing the run.
|
|
33
|
+
*
|
|
34
|
+
* The four job routes are reachable two ways, and both are the same code. From
|
|
35
|
+
* the handle a submit hands back, `job.status()`, when you have the run. From
|
|
36
|
+
* `oc.agent.get(id)` and friends, when all you have is an id off a queue or a
|
|
37
|
+
* webhook — a run outlives the process that submitted it, so an SDK that only
|
|
38
|
+
* spoke to submit handles could not read yesterday's run.
|
|
39
|
+
*/
|
|
40
|
+
import { type AgentEventLevel, type AgentRecord, type AgentRequest, type AgentRun, type AgentSubmission, type Usage } from './_deps/core/index.js';
|
|
41
|
+
import { type ClientContext, type Paged, type Result } from './result.js';
|
|
42
|
+
/**
|
|
43
|
+
* An agent request was rejected before a browser was allocated. Carries every
|
|
44
|
+
* problem at once rather than one per round trip.
|
|
45
|
+
*/
|
|
46
|
+
export declare class InvalidAgentRequestError extends Error {
|
|
47
|
+
readonly name = "InvalidAgentRequestError";
|
|
48
|
+
readonly errors: readonly string[];
|
|
49
|
+
constructor(errors: readonly string[]);
|
|
50
|
+
}
|
|
51
|
+
/** What {@link AgentHandle.events} takes: where to resume, and how much to say. */
|
|
52
|
+
export interface AgentEventOptions {
|
|
53
|
+
/** Resume: everything with a greater `seq`. Omitted starts at the beginning. */
|
|
54
|
+
readonly after?: number;
|
|
55
|
+
readonly level?: AgentEventLevel;
|
|
56
|
+
}
|
|
57
|
+
/** What {@link AgentHandle.results} takes. */
|
|
58
|
+
export interface AgentResultsOptions {
|
|
59
|
+
readonly cursor?: string;
|
|
60
|
+
readonly limit?: number;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* What `POST /v1/agent/{id}/files` answers: the HANDLE and what was stored.
|
|
64
|
+
*
|
|
65
|
+
* `id` is the whole point — it is what an agent's `attachFile` action names,
|
|
66
|
+
* and the model addresses it and never the bytes. It is minted server-side and
|
|
67
|
+
* never chosen by a caller, because `up/<runId>/<fileId>` is a path in a
|
|
68
|
+
* bucket shared by every tenant.
|
|
69
|
+
*
|
|
70
|
+
* `bytes` is the DECODED length, so a caller can confirm the file arrived
|
|
71
|
+
* whole rather than trusting that base64 round-tripped.
|
|
72
|
+
*/
|
|
73
|
+
export interface AgentFile {
|
|
74
|
+
readonly id: string;
|
|
75
|
+
readonly run: string;
|
|
76
|
+
readonly name: string;
|
|
77
|
+
readonly contentType: string;
|
|
78
|
+
readonly bytes: number;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* A run's log, as it happens.
|
|
82
|
+
*
|
|
83
|
+
* ── The paging is invisible and that is the whole design ─────────────────────
|
|
84
|
+
*
|
|
85
|
+
* The wire is a CURSOR TAIL: one request returns everything after a sequence
|
|
86
|
+
* number and ends, which is what lets a stateless Worker serve it and what
|
|
87
|
+
* makes a dropped connection free. Iterating this re-requests from the cursor
|
|
88
|
+
* the previous page ended on and stops when a page comes back `done`, so a
|
|
89
|
+
* caller writes `for await (const record of job.events())` and never sees a
|
|
90
|
+
* page boundary.
|
|
91
|
+
*
|
|
92
|
+
* It backs off while nothing is happening rather than hammering: a run thinking
|
|
93
|
+
* for two minutes about one page produces no records.
|
|
94
|
+
*
|
|
95
|
+
* Iterating twice is allowed and resumes rather than replaying — {@link cursor}
|
|
96
|
+
* is durable state on our side, not a socket. That is the difference from
|
|
97
|
+
* `CrawlHandle`, which refuses a second pass because a held stream genuinely
|
|
98
|
+
* cannot be read again.
|
|
99
|
+
*/
|
|
100
|
+
export declare class AgentEvents implements AsyncIterable<AgentRecord> {
|
|
101
|
+
#private;
|
|
102
|
+
constructor(ctx: ClientContext, id: () => Promise<string>, options?: AgentEventOptions);
|
|
103
|
+
/**
|
|
104
|
+
* The highest `seq` seen. Hand it back as `after` to resume exactly here: the
|
|
105
|
+
* store allocates `seq` gap-free per run, so a consumer that stops cannot
|
|
106
|
+
* miss a row it never saw.
|
|
107
|
+
*/
|
|
108
|
+
get cursor(): number;
|
|
109
|
+
/** The run as its last header frame described it. Undefined before the first. */
|
|
110
|
+
get run(): AgentRun | undefined;
|
|
111
|
+
/**
|
|
112
|
+
* What the run has cost so far, off the header frame the tail opens with.
|
|
113
|
+
* Current rather than final while the run is live, like every other quantity
|
|
114
|
+
* on the row.
|
|
115
|
+
*/
|
|
116
|
+
get usage(): Usage;
|
|
117
|
+
[Symbol.asyncIterator](): AsyncIterator<AgentRecord>;
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* A run on our side, reached by id.
|
|
121
|
+
*
|
|
122
|
+
* Four things a caller does with one: read it, watch it, page its output, stop
|
|
123
|
+
* it. Every one of them is a separate request against a durable row, so a
|
|
124
|
+
* handle is cheap, holds nothing open, and works for a run this process did not
|
|
125
|
+
* submit.
|
|
126
|
+
*/
|
|
127
|
+
export declare class AgentHandle {
|
|
128
|
+
#private;
|
|
129
|
+
constructor(ctx: ClientContext, id: string | (() => Promise<string>));
|
|
130
|
+
/** This run's id. */
|
|
131
|
+
id(): Promise<string>;
|
|
132
|
+
/** Status, plan, findings so far and cost so far. Safe to call in a loop. */
|
|
133
|
+
status(): Promise<Result<AgentRun>>;
|
|
134
|
+
/** Everything the run has done, in order, from a cursor. */
|
|
135
|
+
events(options?: AgentEventOptions): AgentEvents;
|
|
136
|
+
/**
|
|
137
|
+
* The records this run appended, a page at a time, oldest first.
|
|
138
|
+
*
|
|
139
|
+
* Records land DURING the run, so this answers before it has finished — which
|
|
140
|
+
* is the property that makes a killed run still deliver what it bought.
|
|
141
|
+
*/
|
|
142
|
+
results(options?: AgentResultsOptions): Promise<Paged<AgentRecord>>;
|
|
143
|
+
/**
|
|
144
|
+
* Stop the run. Answers the run as it stands, so a caller learns in one round
|
|
145
|
+
* trip whether it had already finished — cancelling a completed run is a
|
|
146
|
+
* no-op and must not read as a failure.
|
|
147
|
+
*
|
|
148
|
+
* Work already performed is still billed. `caps` is the protection; this is
|
|
149
|
+
* the stop button.
|
|
150
|
+
*/
|
|
151
|
+
cancel(): Promise<Result<AgentRun>>;
|
|
152
|
+
/**
|
|
153
|
+
* `POST /v1/agent/{id}/control` — take this run's browser, or hand it back.
|
|
154
|
+
*
|
|
155
|
+
* Taking PAUSES the agent where it stands and leaves the page open;
|
|
156
|
+
* `AgentRun.pause.viewerUrl` on the answer, once the machine has parked, is
|
|
157
|
+
* where a person drives it. Releasing resumes the run from whatever page
|
|
158
|
+
* they left it on — not from where the agent was, because a human who drove
|
|
159
|
+
* may well have navigated, and pretending otherwise is how a click lands on
|
|
160
|
+
* a page nobody read.
|
|
161
|
+
*
|
|
162
|
+
* Answers the RUN and not an acknowledgement, on `cancel`'s terms: the
|
|
163
|
+
* request is durable when this returns, the machine notices within a
|
|
164
|
+
* heartbeat, and a run that had already finished answers with its terminal
|
|
165
|
+
* status rather than throwing — asking for the wheel of a run that just
|
|
166
|
+
* settled is a race a caller cannot avoid.
|
|
167
|
+
*
|
|
168
|
+
* Parked time is not charged to `caps.duration` and is not billed.
|
|
169
|
+
*/
|
|
170
|
+
control(action: 'take' | 'release', who?: string): Promise<Result<AgentRun>>;
|
|
171
|
+
/**
|
|
172
|
+
* `POST /v1/agent/{id}/answer` — reply to a question this run parked on.
|
|
173
|
+
*
|
|
174
|
+
* The question is on `AgentRun.pause` and on the event stream as a
|
|
175
|
+
* `question` record, so a caller that is not watching a socket still finds
|
|
176
|
+
* it. The answer becomes a finding the run cannot forget for the rest of its
|
|
177
|
+
* life, and the run resumes from the step it stopped on.
|
|
178
|
+
*
|
|
179
|
+
* Refused, naming the status, on a run that is not waiting on anything: an
|
|
180
|
+
* answer stored against a run that is not asking would be delivered to
|
|
181
|
+
* whatever it asks next.
|
|
182
|
+
*/
|
|
183
|
+
answer(answer: string, answeredBy?: string): Promise<Result<AgentRun>>;
|
|
184
|
+
/**
|
|
185
|
+
* `POST /v1/agent/{id}/files` — hand this run a file, and get the HANDLE the
|
|
186
|
+
* agent attaches it with.
|
|
187
|
+
*
|
|
188
|
+
* Callable at any point in the run's life, including before it has taken a
|
|
189
|
+
* step: a run id exists the instant the submit is answered, and the bucket is
|
|
190
|
+
* the queue. Refused once the run has ENDED, because the terminal path has
|
|
191
|
+
* already destroyed this run's files and one accepted afterwards would
|
|
192
|
+
* outlive the run it belonged to.
|
|
193
|
+
*
|
|
194
|
+
* `bytes` is the file. Base64 is applied here rather than by the caller
|
|
195
|
+
* because the wire form is a JSON body and encoding it is not a decision a
|
|
196
|
+
* caller should have to make correctly — and getting it wrong produces a
|
|
197
|
+
* file that stores and then attaches corrupt.
|
|
198
|
+
*
|
|
199
|
+
* The file is DESTROYED when the run ends, on every ending. There is no
|
|
200
|
+
* download counterpart and that is deliberate, not missing.
|
|
201
|
+
*/
|
|
202
|
+
addFile(file: {
|
|
203
|
+
readonly name: string;
|
|
204
|
+
readonly bytes: Uint8Array;
|
|
205
|
+
readonly contentType?: string;
|
|
206
|
+
}): Promise<Result<AgentFile>>;
|
|
207
|
+
/**
|
|
208
|
+
* Wait for the run to settle, and answer it.
|
|
209
|
+
*
|
|
210
|
+
* A poll over the row rather than a held socket, so it survives a dropped
|
|
211
|
+
* connection, a restarted process on our side, and a run that takes four
|
|
212
|
+
* hours.
|
|
213
|
+
*/
|
|
214
|
+
settled(): Promise<Result<AgentRun>>;
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* A submitted run.
|
|
218
|
+
*
|
|
219
|
+
* Everything an {@link AgentHandle} does, plus the submit itself: `await` for
|
|
220
|
+
* the finished run, `for await` for the log while it happens. Both are now
|
|
221
|
+
* separate requests against a durable row, so unlike the held-socket version
|
|
222
|
+
* they are not exclusive — doing both is one run, not two.
|
|
223
|
+
*/
|
|
224
|
+
export declare class AgentJob extends AgentHandle implements PromiseLike<Result<AgentRun>>, AsyncIterable<AgentRecord> {
|
|
225
|
+
#private;
|
|
226
|
+
/**
|
|
227
|
+
* The submit response: the id, and that it is queued.
|
|
228
|
+
*
|
|
229
|
+
* A promise, because submitting is a round trip and this object is returned
|
|
230
|
+
* synchronously — `oc.agent(...)` cannot block. Everything else on this class
|
|
231
|
+
* awaits it, so a caller who only wants to fire and forget never has to.
|
|
232
|
+
*/
|
|
233
|
+
readonly submitted: Promise<Result<AgentSubmission>>;
|
|
234
|
+
constructor(ctx: ClientContext, request: AgentRequest);
|
|
235
|
+
/**
|
|
236
|
+
* Await the finished run.
|
|
237
|
+
*
|
|
238
|
+
* Kept, because `const run = await oc.agent({...})` is the line every caller
|
|
239
|
+
* writes first and refusing to support it would make the simple case the
|
|
240
|
+
* awkward one. Memoised: two `await`s on the same job are one poll loop, not
|
|
241
|
+
* two.
|
|
242
|
+
*/
|
|
243
|
+
then<TResult1 = Result<AgentRun>, TResult2 = never>(onfulfilled?: ((value: Result<AgentRun>) => TResult1 | PromiseLike<TResult1>) | null, onrejected?: ((reason: unknown) => TResult2 | PromiseLike<TResult2>) | null): PromiseLike<TResult1 | TResult2>;
|
|
244
|
+
/**
|
|
245
|
+
* Iterate the run's log. `for await (const record of oc.agent({...}))` is the
|
|
246
|
+
* other line callers write, and it is {@link AgentHandle.events} underneath —
|
|
247
|
+
* the cursor tail, not a socket.
|
|
248
|
+
*/
|
|
249
|
+
[Symbol.asyncIterator](): AsyncIterator<AgentRecord>;
|
|
250
|
+
}
|
|
251
|
+
/**
|
|
252
|
+
* Submit an agent run.
|
|
253
|
+
*
|
|
254
|
+
* The request is validated with core's own validator, which is the same check
|
|
255
|
+
* the API applies — so a JavaScript caller who omits `caps` is refused here,
|
|
256
|
+
* in their own process, with the reason, rather than by a 4xx after a round
|
|
257
|
+
* trip. TypeScript callers cannot omit it at all.
|
|
258
|
+
*/
|
|
259
|
+
export declare function agent(ctx: ClientContext, request: AgentRequest): AgentJob;
|
|
260
|
+
/**
|
|
261
|
+
* `oc.agent` — callable, and the namespace for the job routes.
|
|
262
|
+
*
|
|
263
|
+
* Callable because submitting is what the capability IS, and `oc.agent.submit`
|
|
264
|
+
* would be a second spelling of it. A namespace as well because the registry
|
|
265
|
+
* declares seven more rows against the same run and an id from a webhook has
|
|
266
|
+
* to reach them: `oc.agent.get(id)` is the same request `job.status()` makes.
|
|
267
|
+
*
|
|
268
|
+
* **Every member is named after its row's last segment**, which is why the
|
|
269
|
+
* file route is `files` and not `addFile`: `test/registry.test.ts` walks the
|
|
270
|
+
* dotted capability name — `agent.files` — straight onto this object, and a
|
|
271
|
+
* member spelled differently is a row the coverage test reports as
|
|
272
|
+
* unreachable. `AgentHandle.addFile` keeps the readable name, because a handle
|
|
273
|
+
* already knows its run and reads as a verb on it.
|
|
274
|
+
*/
|
|
275
|
+
export interface AgentApi {
|
|
276
|
+
(request: AgentRequest): AgentJob;
|
|
277
|
+
/** A handle on a run this process did not submit. */
|
|
278
|
+
job(id: string): AgentHandle;
|
|
279
|
+
/** `GET /v1/agent/{id}` — where one run has got to. */
|
|
280
|
+
get(id: string): Promise<Result<AgentRun>>;
|
|
281
|
+
/** `GET /v1/agent/{id}/events` — the log, from a cursor. */
|
|
282
|
+
events(id: string, options?: AgentEventOptions): AgentEvents;
|
|
283
|
+
/** `GET /v1/agent/{id}/results` — the records it appended. */
|
|
284
|
+
results(id: string, options?: AgentResultsOptions): Promise<Paged<AgentRecord>>;
|
|
285
|
+
/** `POST /v1/agent/{id}/cancel` — stop spending. */
|
|
286
|
+
cancel(id: string): Promise<Result<AgentRun>>;
|
|
287
|
+
/**
|
|
288
|
+
* `POST /v1/agent/{id}/control` — take this run's browser, or hand it back.
|
|
289
|
+
*
|
|
290
|
+
* Taking pauses the agent where it stands and leaves the page open; the
|
|
291
|
+
* answer carries `AgentRun.pause.viewerUrl` once the machine has parked.
|
|
292
|
+
* Releasing resumes from whatever page the person left it on.
|
|
293
|
+
*/
|
|
294
|
+
control(id: string, action: 'take' | 'release', who?: string): Promise<Result<AgentRun>>;
|
|
295
|
+
/**
|
|
296
|
+
* `POST /v1/agent/{id}/answer` — reply to a question this run parked on.
|
|
297
|
+
*
|
|
298
|
+
* Refused, naming the status, on a run that is not waiting on anything.
|
|
299
|
+
*/
|
|
300
|
+
answer(id: string, answer: string, answeredBy?: string): Promise<Result<AgentRun>>;
|
|
301
|
+
/**
|
|
302
|
+
* `POST /v1/agent/{id}/files` — hand a run a file, at any point in its life.
|
|
303
|
+
*
|
|
304
|
+
* Answers the HANDLE, which is what the agent's `attachFile` action takes.
|
|
305
|
+
* The file is destroyed when the run ends. There is no download counterpart.
|
|
306
|
+
*/
|
|
307
|
+
files(id: string, file: {
|
|
308
|
+
readonly name: string;
|
|
309
|
+
readonly bytes: Uint8Array;
|
|
310
|
+
readonly contentType?: string;
|
|
311
|
+
}): Promise<Result<AgentFile>>;
|
|
312
|
+
}
|
|
313
|
+
export declare function createAgentApi(ctx: ClientContext): AgentApi;
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which capabilities this SDK will actually let a caller reach.
|
|
3
|
+
*
|
|
4
|
+
* The registry in `@outcrawl/core` answers "does this capability have a route".
|
|
5
|
+
* It deliberately does not answer "does anything run behind that route", and
|
|
6
|
+
* those are different questions: `monitors.*` has a registry row, a matched
|
|
7
|
+
* route, a request validator, a Postgres table, a tenant-scoped store and, as
|
|
8
|
+
* of tonight, a working scheduler — and the row a customer creates still never
|
|
9
|
+
* reaches any of it, because the API writes to memory and the scheduler reads
|
|
10
|
+
* Postgres. `POST /v1/monitors` answers `201` for a monitor that will never be
|
|
11
|
+
* checked and never notify anybody.
|
|
12
|
+
*
|
|
13
|
+
* That is the worst failure a client can have, because it is indistinguishable
|
|
14
|
+
* from success. It fails at the customer's integration, weeks later, as "my
|
|
15
|
+
* monitor never fired" — not at the call. So this table is the seam between
|
|
16
|
+
* "declared" and "served", and it is checked at the one place every REST call
|
|
17
|
+
* in this package passes through.
|
|
18
|
+
*
|
|
19
|
+
* THE TEST FOR ADDING A ROW HERE — and it is not "how broken does this feel":
|
|
20
|
+
*
|
|
21
|
+
* Does the capability DO ANYTHING for the caller as it stands?
|
|
22
|
+
*
|
|
23
|
+
* If no, it is inert and belongs in this table. A monitor held in one machine's
|
|
24
|
+
* memory is inert: the scheduler reads Postgres and will never see it, so
|
|
25
|
+
* `create` is a promise that is never kept, and refusing is the honest answer.
|
|
26
|
+
* If yes, it stays `null` and its limits are DOCUMENTED, however uncomfortable
|
|
27
|
+
* they are. A profile held in that same memory genuinely works — it pins the
|
|
28
|
+
* identity and persists the cookies for as long as that machine is up — so
|
|
29
|
+
* refusing it would break something a customer can use today. Same defect
|
|
30
|
+
* underneath, opposite correct response, and what separates them is whether the
|
|
31
|
+
* feature does anything, not how bad the underlying state is. Getting this
|
|
32
|
+
* backwards ships an SDK that refuses working features, which is the same
|
|
33
|
+
* defect as one that offers broken features, pointed the other way.
|
|
34
|
+
*
|
|
35
|
+
* The field is REQUIRED for every capability and an available one declines it
|
|
36
|
+
* with an explicit `null`. That is the whole point: `satisfies
|
|
37
|
+
* Record<CapabilityName, …>` means a row added to the registry does not compile
|
|
38
|
+
* here until somebody states whether anything serves it. An optional field
|
|
39
|
+
* would have defaulted the answer to "available", which is the exact shape of
|
|
40
|
+
* the defect this file exists to prevent — the SDK claiming a capability the
|
|
41
|
+
* server does not have.
|
|
42
|
+
*/
|
|
43
|
+
import { type CapabilityName } from './_deps/core/index.js';
|
|
44
|
+
/** Why a declared capability cannot be reached, and when it will be. */
|
|
45
|
+
export interface Unavailable {
|
|
46
|
+
/**
|
|
47
|
+
* What is not built, in the words {@link CapabilityUnavailableError} uses
|
|
48
|
+
* after "no": `monitor scheduler` reads as "no monitor scheduler is
|
|
49
|
+
* configured".
|
|
50
|
+
*/
|
|
51
|
+
readonly missing: string;
|
|
52
|
+
/**
|
|
53
|
+
* The rest of the story, appended to the error message verbatim. States what
|
|
54
|
+
* does exist, what does not, and when it will — with no invented date, since
|
|
55
|
+
* a date nobody committed to is another optional field filled in by guessing.
|
|
56
|
+
*/
|
|
57
|
+
readonly detail: string;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* `null` means served: the capability is reachable and the call goes out.
|
|
61
|
+
*
|
|
62
|
+
* Anything else is refused here, in the caller's own process, before a request
|
|
63
|
+
* is built. Refusing locally rather than letting the server answer is right for
|
|
64
|
+
* exactly one reason — the server answers these successfully today.
|
|
65
|
+
*/
|
|
66
|
+
export declare const CAPABILITY_AVAILABILITY: Readonly<Record<CapabilityName, Unavailable | null>>;
|
|
67
|
+
/**
|
|
68
|
+
* Refuse a capability nothing serves.
|
|
69
|
+
*
|
|
70
|
+
* Throws {@link CapabilityUnavailableError} — the existing error, with the
|
|
71
|
+
* existing `capability_unavailable` code, rather than a second class carrying
|
|
72
|
+
* the same code. A caller already branching on that code for an unconfigured
|
|
73
|
+
* worker needs no new case, and `isOutcrawlError` keeps working.
|
|
74
|
+
*
|
|
75
|
+
* FAILS CLOSED on a capability the table does not declare. `satisfies` catches
|
|
76
|
+
* a missing row at compile time, but it cannot help a name that arrives at
|
|
77
|
+
* runtime — from a newer server, or a hand-built `resolveRequest` call through
|
|
78
|
+
* the published transport export. The first version of this function read
|
|
79
|
+
* `undefined` for such a name, took the `!== null` branch and then threw a
|
|
80
|
+
* `TypeError` about reading `missing` of undefined: refused, technically, but
|
|
81
|
+
* with an untyped error carrying no code, which every caller branching on
|
|
82
|
+
* `code` would have dropped into its generic handler. Undeclared now refuses
|
|
83
|
+
* by name, because "unknown means allowed" is how a client ends up claiming a
|
|
84
|
+
* capability nobody implemented.
|
|
85
|
+
*/
|
|
86
|
+
export declare function assertAvailable(capability: CapabilityName): void;
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Opening a browser.
|
|
3
|
+
*
|
|
4
|
+
* Two options exist and no more: a profile to pin identity and storage to, and
|
|
5
|
+
* geography. Everything else that shapes an identity — seed, chrome-or-brave,
|
|
6
|
+
* fingerprint, timezone, languages, WebRTC address — is minted by us and pinned
|
|
7
|
+
* for the profile's life, because every one of them is a way to build an
|
|
8
|
+
* identity that cannot exist, and incoherence is what gets caught, not any
|
|
9
|
+
* single value.
|
|
10
|
+
*
|
|
11
|
+
* So a script that passes `timezoneId` is refused here, loudly, by name. That
|
|
12
|
+
* is not pedantry: Playwright applies `timezoneId`, `locale`, `userAgent` and
|
|
13
|
+
* `viewport` itself over per-page CDP *after* the context exists, and the
|
|
14
|
+
* worker refuses the corresponding `Emulation.*` calls because those values are
|
|
15
|
+
* hard-pinned to the exit. Failing in the caller's process with the conflict
|
|
16
|
+
* named beats failing halfway through a checkout with a CDP error.
|
|
17
|
+
*/
|
|
18
|
+
import type { ExitSpec, ExitTarget } from './_deps/core/index.js';
|
|
19
|
+
import type { HandsFactory, RawPage } from './hands.js';
|
|
20
|
+
import { OutcrawlPage } from './page.js';
|
|
21
|
+
import type { Connection, RawInputMark, Transport } from './transport.js';
|
|
22
|
+
/**
|
|
23
|
+
* A client-side identity override was refused. Names the option and what
|
|
24
|
+
* already pins it, so the fix is obvious rather than a support ticket.
|
|
25
|
+
*/
|
|
26
|
+
export declare class IdentityOverrideError extends Error {
|
|
27
|
+
readonly name = "IdentityOverrideError";
|
|
28
|
+
/** The option that was passed. */
|
|
29
|
+
readonly option: string;
|
|
30
|
+
/** What pins the value instead. */
|
|
31
|
+
readonly pinnedBy: string;
|
|
32
|
+
constructor(option: string, pinnedBy: string);
|
|
33
|
+
}
|
|
34
|
+
/** The complete set of options a caller may pass. */
|
|
35
|
+
export interface BrowserOptions {
|
|
36
|
+
/** A profile id or label. Supplies pinned identity and logged-in state. */
|
|
37
|
+
profile?: string;
|
|
38
|
+
/**
|
|
39
|
+
* `outcrawl://<country>[/<region>[/<city>]]`, or the same as a structured
|
|
40
|
+
* target. Country, region and city. Nothing else — an exit chosen by ASN or
|
|
41
|
+
* postcode is a handful of IPs whose reputation we cannot manage.
|
|
42
|
+
*/
|
|
43
|
+
proxy?: ExitSpec;
|
|
44
|
+
/**
|
|
45
|
+
* Session recording, ON BY DEFAULT. Pass `false` to decline it.
|
|
46
|
+
*
|
|
47
|
+
* Recording a `/connect` session is automatic and this is the only way out.
|
|
48
|
+
* Absent means RECORDED, and so does `true`; only an explicit `false`
|
|
49
|
+
* declines. It travels as `?record=false` on the allocation request and is
|
|
50
|
+
* carried to the machine inside the signed ticket, so no relay hop can
|
|
51
|
+
* strip it and turn a refusal back into a recording.
|
|
52
|
+
*
|
|
53
|
+
* Honoured before anything is armed. A declined session never has a
|
|
54
|
+
* recorder attached to its pages, so there is no stored replay, no
|
|
55
|
+
* `replayUrl` on the session, and `sessions.export` answers `404`. It is not
|
|
56
|
+
* a recording deleted afterwards — the bytes never existed. The session is
|
|
57
|
+
* billed exactly as a recorded one is.
|
|
58
|
+
*
|
|
59
|
+
* Afterwards the session says which it was: `SessionSummary.recordingDeclined`
|
|
60
|
+
* is `true`, which is what separates a refusal from a recorder that failed.
|
|
61
|
+
*
|
|
62
|
+
* NOT an identity override, which is why it is allowed where `userAgent` and
|
|
63
|
+
* `timezoneId` are refused: it decides what we keep about the session, not
|
|
64
|
+
* what the browser presents to a site.
|
|
65
|
+
*/
|
|
66
|
+
record?: boolean;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Refuse client-side identity overrides before a socket is opened.
|
|
70
|
+
*
|
|
71
|
+
* Unknown options are refused too, not ignored. A silently dropped option is
|
|
72
|
+
* indistinguishable from one that worked, and the customer finds out when a
|
|
73
|
+
* session is blocked rather than when they wrote the line.
|
|
74
|
+
*/
|
|
75
|
+
export declare function assertNoIdentityOverride(options: object, where: string): void;
|
|
76
|
+
/** Normalise the one geography option into the structured form. */
|
|
77
|
+
export declare function resolveExit(proxy: ExitSpec): ExitTarget;
|
|
78
|
+
export interface BrowserDeps {
|
|
79
|
+
transport: Transport;
|
|
80
|
+
hands: HandsFactory;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* A live browser. One tenant-scoped CDP connection: it can only ever see its
|
|
84
|
+
* own context.
|
|
85
|
+
*/
|
|
86
|
+
export declare class Browser {
|
|
87
|
+
#private;
|
|
88
|
+
constructor(connection: Connection, deps: BrowserDeps);
|
|
89
|
+
get sessionId(): string;
|
|
90
|
+
/**
|
|
91
|
+
* Every raw-input call this session made, in order. The same marks are sent
|
|
92
|
+
* to the session record, so a run is flagged both here and in the replay.
|
|
93
|
+
*/
|
|
94
|
+
get rawCalls(): readonly RawInputMark[];
|
|
95
|
+
pages(): readonly OutcrawlPage<RawPage>[];
|
|
96
|
+
/**
|
|
97
|
+
* `newPage()` takes no options, deliberately. Playwright's does, and the ones
|
|
98
|
+
* it takes are exactly the identity overrides this product cannot honour —
|
|
99
|
+
* so they are refused by name rather than quietly dropped.
|
|
100
|
+
*/
|
|
101
|
+
newPage(options?: Record<string, never>): Promise<OutcrawlPage<RawPage>>;
|
|
102
|
+
close(): Promise<void>;
|
|
103
|
+
}
|
|
104
|
+
export declare function openBrowser(deps: BrowserDeps, options?: BrowserOptions, extra?: {
|
|
105
|
+
attachTo?: string;
|
|
106
|
+
signal?: AbortSignal;
|
|
107
|
+
}): Promise<Browser>;
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `Outcrawl` — the client.
|
|
3
|
+
*
|
|
4
|
+
* Three rules, and everything else follows from them:
|
|
5
|
+
*
|
|
6
|
+
* 1. **The SDK is optional.** An unmodified Playwright script pointed at
|
|
7
|
+
* `wss://outcrawl.ai/connect` keeps working forever. This is ergonomics,
|
|
8
|
+
* never the only door.
|
|
9
|
+
* 2. **Good defaults, not options.** Identity, seed, proxy and fingerprint are
|
|
10
|
+
* minted, not chosen. Anything that can be configured wrong eventually is.
|
|
11
|
+
* 3. **Human input is the default, not a flag.** `page.click()` goes through
|
|
12
|
+
* the measured human model; `page.raw.click()` is the escape hatch, and it
|
|
13
|
+
* is recorded.
|
|
14
|
+
*
|
|
15
|
+
* Every REST method here is addressed by a capability name from
|
|
16
|
+
* `@outcrawl/core`. Method and path come from the registry row, so this client
|
|
17
|
+
* cannot reach a route the registry does not declare, and a capability added to
|
|
18
|
+
* the registry has an obvious, single place to land.
|
|
19
|
+
*/
|
|
20
|
+
import type { CreditBalance, ScrapeResult, UsageQuery, UsageReport } from './_deps/core/index.js';
|
|
21
|
+
import { type AgentApi } from './agent.js';
|
|
22
|
+
import { type Browser, type BrowserOptions } from './browser.js';
|
|
23
|
+
import { type HandsFactory } from './hands.js';
|
|
24
|
+
import { Integrations } from './integrations.js';
|
|
25
|
+
import { Monitors } from './monitors.js';
|
|
26
|
+
import { Profiles } from './profiles.js';
|
|
27
|
+
import { Rules } from './rules.js';
|
|
28
|
+
import { Secrets } from './secrets.js';
|
|
29
|
+
import { type Result } from './result.js';
|
|
30
|
+
import { type CrawlHandle, type CrawlOptions, type ScrapeOptions, type SearchHits, type SearchOptions } from './scrape.js';
|
|
31
|
+
import { Sessions } from './sessions.js';
|
|
32
|
+
import { type Transport } from './transport.js';
|
|
33
|
+
/**
|
|
34
|
+
* The two environment variables every Outcrawl surface reads.
|
|
35
|
+
*
|
|
36
|
+
* One spelling across the CLI, the MCP server, this SDK and the Python one: a
|
|
37
|
+
* customer moving between them must not have to rename their environment.
|
|
38
|
+
* `@outcrawl/cli`'s `config.ts` declares the same two names, and the MCP
|
|
39
|
+
* server read `OUTCRAWL_BASE_URL` for a while — a second spelling for one
|
|
40
|
+
* concept, which is how a key the CLI reads and the MCP does not comes about.
|
|
41
|
+
*/
|
|
42
|
+
export declare const API_KEY_ENV = "OUTCRAWL_API_KEY";
|
|
43
|
+
export declare const API_URL_ENV = "OUTCRAWL_API_URL";
|
|
44
|
+
export interface OutcrawlOptions {
|
|
45
|
+
/**
|
|
46
|
+
* Required unless a `transport` is supplied, or `OUTCRAWL_API_KEY` is in the
|
|
47
|
+
* environment.
|
|
48
|
+
*/
|
|
49
|
+
apiKey?: string;
|
|
50
|
+
/** Defaults to `OUTCRAWL_API_URL`, and then to `https://outcrawl.ai`. */
|
|
51
|
+
baseUrl?: string;
|
|
52
|
+
/**
|
|
53
|
+
* Replaces the whole network layer. The reason the SDK is testable end to end
|
|
54
|
+
* with no network and no browser.
|
|
55
|
+
*/
|
|
56
|
+
transport?: Transport;
|
|
57
|
+
/**
|
|
58
|
+
* The human-input driver. Defaults to the built-in one, which already uses
|
|
59
|
+
* `agenthands` for the scroll model; replace it to drive input some other way.
|
|
60
|
+
* Not a way to turn human input off — there is no such option, because opt-in
|
|
61
|
+
* stealth is stealth nobody uses.
|
|
62
|
+
*
|
|
63
|
+
* The factory is handed a {@link HandsBinding}, not a bare persona: a driver
|
|
64
|
+
* also needs the browser's own wheel entry point, which is a property of the
|
|
65
|
+
* session rather than of the call.
|
|
66
|
+
*/
|
|
67
|
+
hands?: HandsFactory;
|
|
68
|
+
/** Injected fetch, for the default transport. */
|
|
69
|
+
fetch?: typeof globalThis.fetch;
|
|
70
|
+
}
|
|
71
|
+
export declare class Outcrawl {
|
|
72
|
+
#private;
|
|
73
|
+
readonly transport: Transport;
|
|
74
|
+
/**
|
|
75
|
+
* Callable, and the namespace for the four job routes. `oc.agent(...)`
|
|
76
|
+
* submits; `oc.agent.get(id)` and friends reach a run by id.
|
|
77
|
+
*/
|
|
78
|
+
readonly agent: AgentApi;
|
|
79
|
+
readonly profiles: Profiles;
|
|
80
|
+
/** The secrets store. No method returns a value; see `./secrets.ts`. */
|
|
81
|
+
readonly secrets: Secrets;
|
|
82
|
+
/**
|
|
83
|
+
* The workspace rules, and which of them are ENFORCED rather than advice.
|
|
84
|
+
* Read `class` and `unenforceable`, not just `text`; see `./rules.ts`.
|
|
85
|
+
*/
|
|
86
|
+
readonly rules: Rules;
|
|
87
|
+
/**
|
|
88
|
+
* What a run can reach OUTSIDE the browser, once a submit names it.
|
|
89
|
+
* Connecting is not granting; see `./integrations.ts`.
|
|
90
|
+
*/
|
|
91
|
+
readonly integrations: Integrations;
|
|
92
|
+
readonly sessions: Sessions;
|
|
93
|
+
readonly monitors: Monitors;
|
|
94
|
+
constructor(options?: OutcrawlOptions);
|
|
95
|
+
scrape(url: string, options?: ScrapeOptions): Promise<Result<ScrapeResult>>;
|
|
96
|
+
crawl(url: string, options?: CrawlOptions): Promise<CrawlHandle>;
|
|
97
|
+
search(query: string, options?: SearchOptions): Promise<SearchHits>;
|
|
98
|
+
/**
|
|
99
|
+
* Open a browser. With no profile the identity is ephemeral; with one it is
|
|
100
|
+
* pinned, along with its storage, and the profile is leased for the duration
|
|
101
|
+
* — a second concurrent open throws `ProfileInUseError` rather than silently
|
|
102
|
+
* forking cookies.
|
|
103
|
+
*/
|
|
104
|
+
browser(options?: BrowserOptions): Promise<Browser>;
|
|
105
|
+
/**
|
|
106
|
+
* The same numbers already attached to each call, aggregated. Nothing appears
|
|
107
|
+
* here that was not visible on the call that caused it.
|
|
108
|
+
*/
|
|
109
|
+
usage(query: UsageQuery): Promise<Result<UsageReport>>;
|
|
110
|
+
/**
|
|
111
|
+
* The credit balance, summed by the database off the append-only ledger and
|
|
112
|
+
* the meter — `granted - spent`, with what lapsed reported separately so
|
|
113
|
+
* "where did my 4,000 credits go" has an answer other than a single number.
|
|
114
|
+
*
|
|
115
|
+
* **Every figure is a decimal STRING and must stay one.** They are credits.
|
|
116
|
+
* Parsing them to a `number` reintroduces binary floating point to money,
|
|
117
|
+
* which is the whole reason the wire carries text. Compare and add them with
|
|
118
|
+
* a decimal library, or pass them straight through to your UI.
|
|
119
|
+
*
|
|
120
|
+
* `exhausted` is REPORTED, NOT ENFORCED, and this is the trap worth naming:
|
|
121
|
+
* nothing refuses work when the granted pool empties. Admission is decided
|
|
122
|
+
* against the plan's monthly allowance, and a grant deliberately does not
|
|
123
|
+
* move that. So `exhausted: true` means "the pool you were granted is spent",
|
|
124
|
+
* never "the next call will be refused" — treat it as a signal to top up, not
|
|
125
|
+
* as a gate you can rely on.
|
|
126
|
+
*/
|
|
127
|
+
credits(): Promise<Result<CreditBalance>>;
|
|
128
|
+
}
|