@outcrawl/sdk 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +198 -0
- package/dist/index.js +2493 -0
- package/dist/types/_deps/core/agent-alias.d.ts +219 -0
- package/dist/types/_deps/core/brand.d.ts +62 -0
- package/dist/types/_deps/core/certificate.d.ts +100 -0
- package/dist/types/_deps/core/cron.d.ts +116 -0
- package/dist/types/_deps/core/errors.d.ts +340 -0
- package/dist/types/_deps/core/index.d.ts +17 -0
- package/dist/types/_deps/core/money.d.ts +108 -0
- package/dist/types/_deps/core/registry.d.ts +466 -0
- package/dist/types/_deps/core/rules.d.ts +237 -0
- package/dist/types/_deps/core/secrets.d.ts +278 -0
- package/dist/types/_deps/core/types.d.ts +1677 -0
- package/dist/types/_deps/integrations/connector.d.ts +153 -0
- package/dist/types/_deps/replay/events.d.ts +1004 -0
- package/dist/types/agent.d.ts +313 -0
- package/dist/types/availability.d.ts +86 -0
- package/dist/types/browser.d.ts +107 -0
- package/dist/types/client.d.ts +128 -0
- package/dist/types/hands.d.ts +259 -0
- package/dist/types/index.d.ts +111 -0
- package/dist/types/integrations.d.ts +63 -0
- package/dist/types/monitors.d.ts +24 -0
- package/dist/types/page.d.ts +96 -0
- package/dist/types/profiles.d.ts +26 -0
- package/dist/types/result.d.ts +90 -0
- package/dist/types/rules.d.ts +41 -0
- package/dist/types/scrape.d.ts +68 -0
- package/dist/types/secrets.d.ts +36 -0
- package/dist/types/sessions.d.ts +124 -0
- package/dist/types/transport.d.ts +250 -0
- package/package.json +70 -0
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Turning a wire response into an SDK result.
|
|
3
|
+
*
|
|
4
|
+
* One rule, applied everywhere: **every result carries `usage`.** Not a metrics
|
|
5
|
+
* endpoint — the number that will appear on the bill, attached to the thing
|
|
6
|
+
* that caused it. Where the core type already has a `usage` field it is used as
|
|
7
|
+
* it arrives; where it does not, the API returns it in an envelope and it is
|
|
8
|
+
* attached here.
|
|
9
|
+
*
|
|
10
|
+
* Attached non-enumerably, so a decorated array is still the array a caller
|
|
11
|
+
* expects from `JSON.stringify` and from a deep equality assertion, and
|
|
12
|
+
* `hits.usage` still works.
|
|
13
|
+
*
|
|
14
|
+
* Nothing here invents a number. A response with no usage at all gets `{}`,
|
|
15
|
+
* which reads as "nothing to report" rather than as a measured zero.
|
|
16
|
+
*/
|
|
17
|
+
import { type Usage } from './_deps/core/index.js';
|
|
18
|
+
import type { HandsFactory } from './hands.js';
|
|
19
|
+
import type { Transport } from './transport.js';
|
|
20
|
+
/** What every capability method on the client is given. */
|
|
21
|
+
export interface ClientContext {
|
|
22
|
+
transport: Transport;
|
|
23
|
+
hands: HandsFactory;
|
|
24
|
+
}
|
|
25
|
+
/** A single result, with usage guaranteed. */
|
|
26
|
+
export type Result<T> = T & {
|
|
27
|
+
readonly usage: Usage;
|
|
28
|
+
};
|
|
29
|
+
/**
|
|
30
|
+
* A list result. The array is the ergonomic surface; `cursor` and `usage` ride
|
|
31
|
+
* alongside it, out of the way of iteration.
|
|
32
|
+
*
|
|
33
|
+
* `cursor` absent means the last page — never `null`, because absence and null
|
|
34
|
+
* would be two ways to spell one state.
|
|
35
|
+
*/
|
|
36
|
+
export type Paged<T> = readonly T[] & {
|
|
37
|
+
readonly cursor: string | undefined;
|
|
38
|
+
readonly usage: Usage;
|
|
39
|
+
};
|
|
40
|
+
/** Attach usage to an array without making it an element or a visible key. */
|
|
41
|
+
export declare function withUsage<T>(items: readonly T[], usage: Usage): readonly T[] & {
|
|
42
|
+
readonly usage: Usage;
|
|
43
|
+
};
|
|
44
|
+
/** Decorate a response whose core type already carries `usage` inline. */
|
|
45
|
+
export declare function inlineUsage<T extends object>(body: T): Result<T>;
|
|
46
|
+
/**
|
|
47
|
+
* Unwrap `{ data, usage }` for the capabilities whose core type has no usage of
|
|
48
|
+
* its own — profiles, monitors, session exports and deletes.
|
|
49
|
+
*
|
|
50
|
+
* A bare body is accepted too, so a 204 delete or an unenveloped response is
|
|
51
|
+
* not a crash.
|
|
52
|
+
*
|
|
53
|
+
* THE ITEM'S OWN `usage` WINS, and that condition is load-bearing rather than
|
|
54
|
+
* defensive. `AgentRun` legitimately carries both a `data` and a `usage` field
|
|
55
|
+
* — the tokens, the inference time and the cache verdict for the run — and it
|
|
56
|
+
* arrives inside this envelope, measured on 2026-09-08 against production:
|
|
57
|
+
*
|
|
58
|
+
* GET /v1/agent/{id}
|
|
59
|
+
* -> { data: { …, usage: { inputTokens: 3886, outputTokens: 4817, … } },
|
|
60
|
+
* usage: {} }
|
|
61
|
+
*
|
|
62
|
+
* The earlier version defined `usage` unconditionally, so every agent read
|
|
63
|
+
* through this SDK — `agent.get`, `job.status()`, `await job`, `cancel`,
|
|
64
|
+
* `control`, `answer` — replaced the run's real usage with the envelope's
|
|
65
|
+
* empty one, and a customer reconciling an agent bill saw `{}`. Found by
|
|
66
|
+
* running the packed tarball against production beside the Python client,
|
|
67
|
+
* which printed the tokens the TypeScript one had already discarded.
|
|
68
|
+
*/
|
|
69
|
+
export declare function unwrapItem<T extends object>(body: unknown): Result<T>;
|
|
70
|
+
/** Turn core's `ListResponse<T>` into the array form the SDK hands back. */
|
|
71
|
+
export declare function pagedFrom<T>(body: unknown): Paged<T>;
|
|
72
|
+
/** Confirmation that a delete happened, with what it cost. */
|
|
73
|
+
export interface Deleted {
|
|
74
|
+
id: string;
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* A delete confirmation. The id is echoed back from the request when the API
|
|
78
|
+
* answers 204 with no body, so the result is the same shape either way and a
|
|
79
|
+
* caller never has to branch on how empty the response was.
|
|
80
|
+
*/
|
|
81
|
+
export declare function deletedFrom(body: unknown, id: string): Result<Deleted>;
|
|
82
|
+
/**
|
|
83
|
+
* Query objects go on the URL, so every value has to be a string. Undefined
|
|
84
|
+
* fields are dropped rather than sent as the string "undefined".
|
|
85
|
+
*
|
|
86
|
+
* Takes `object`, not `Record<string, unknown>`: core's query interfaces have
|
|
87
|
+
* no index signature, and widening them at every call site with a cast would
|
|
88
|
+
* put a cast on the one path where a wrong key matters.
|
|
89
|
+
*/
|
|
90
|
+
export declare function queryOf(query: object): Record<string, string>;
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Rules — what a customer forbids, in their own words, and which of those we
|
|
3
|
+
* actually ENFORCE.
|
|
4
|
+
*
|
|
5
|
+
* The two methods return the same thing, and that is the design rather than a
|
|
6
|
+
* shortcut: {@link Rules.set} answers the classification of the rules it just
|
|
7
|
+
* stored, so a caller learns the split on the call that set them instead of on
|
|
8
|
+
* a later read they might never make.
|
|
9
|
+
*
|
|
10
|
+
* ── READ `class`, NOT `text` ─────────────────────────────────────────────────
|
|
11
|
+
*
|
|
12
|
+
* `class: 'enforced'` means the sentence compiled to a gate `@outcrawl/runtime`
|
|
13
|
+
* checks before every action, and a matching action is REFUSED before it
|
|
14
|
+
* reaches the page. `class: 'guidance'` means it is a line in the agent's
|
|
15
|
+
* prompt, which a model weighs against everything else it was told and may
|
|
16
|
+
* trade away under pressure.
|
|
17
|
+
*
|
|
18
|
+
* Three kinds compile to gates: a spend ceiling, a site allow/deny list, and a
|
|
19
|
+
* confirmation requirement on something irreversible. Anything else is
|
|
20
|
+
* guidance. When a sentence LOOKS enforceable and did not compile —
|
|
21
|
+
* an amount with no currency, a host with no direction, two constraints in one
|
|
22
|
+
* sentence — `unenforceable` says so in words. **That field is the one to
|
|
23
|
+
* check**: it is present exactly when we suspect a customer believes a rule is
|
|
24
|
+
* enforced and it is not.
|
|
25
|
+
*/
|
|
26
|
+
import { type RulesView } from './_deps/core/index.js';
|
|
27
|
+
import { type ClientContext, type Result } from './result.js';
|
|
28
|
+
export declare class Rules {
|
|
29
|
+
#private;
|
|
30
|
+
constructor(ctx: ClientContext);
|
|
31
|
+
/** The workspace rules and, per rule, whether it is enforced or guidance. */
|
|
32
|
+
get(): Promise<Result<RulesView>>;
|
|
33
|
+
/**
|
|
34
|
+
* REPLACE the workspace rules, and read the classification back.
|
|
35
|
+
*
|
|
36
|
+
* Replace and not append: there is no add-one route, because a rule set is
|
|
37
|
+
* read and rendered whole and a partial write is a workspace bound by half
|
|
38
|
+
* of two sets. Send the rules you want.
|
|
39
|
+
*/
|
|
40
|
+
set(rules: readonly string[]): Promise<Result<RulesView>>;
|
|
41
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scrape, crawl and search — the three capabilities that read pages rather than
|
|
3
|
+
* drive them.
|
|
4
|
+
*
|
|
5
|
+
* All three go through the registry: the method and path come from the
|
|
6
|
+
* capability row, never written out here.
|
|
7
|
+
*/
|
|
8
|
+
import type { CrawlJob, CrawlPage, CrawlRequest, ScrapeRequest, ScrapeResult, SearchHit, SearchRequest, Usage } from './_deps/core/index.js';
|
|
9
|
+
import { type ClientContext, type Result } from './result.js';
|
|
10
|
+
export type ScrapeOptions = Omit<ScrapeRequest, 'url'>;
|
|
11
|
+
export type CrawlOptions = Omit<CrawlRequest, 'url'>;
|
|
12
|
+
export type SearchOptions = Omit<SearchRequest, 'query'>;
|
|
13
|
+
/** Search results, with the usage they cost attached to the set. */
|
|
14
|
+
export type SearchHits = readonly SearchHit[] & {
|
|
15
|
+
readonly usage: Usage;
|
|
16
|
+
};
|
|
17
|
+
/**
|
|
18
|
+
* Fetch one URL and return clean content in the requested formats.
|
|
19
|
+
*
|
|
20
|
+
* Every request leaves through the browser's own network stack — same JA4, same
|
|
21
|
+
* HTTP/2 profile, same cookie jar — so there is no cheap-fetch-then-escalate
|
|
22
|
+
* path to burn an exit's reputation on a request that was already flagged. The
|
|
23
|
+
* saving is in stopping early, which is what `metadata.escalated` reports.
|
|
24
|
+
*/
|
|
25
|
+
export declare function scrape(ctx: ClientContext, url: string, options?: ScrapeOptions): Promise<Result<ScrapeResult>>;
|
|
26
|
+
/**
|
|
27
|
+
* Search, optionally with every hit rendered through the scrape pipeline.
|
|
28
|
+
*
|
|
29
|
+
* Thin by design — the index is not our fight. What we add is that results come
|
|
30
|
+
* back rendered by us, which is the one thing the search vendors do not do.
|
|
31
|
+
*/
|
|
32
|
+
export declare function search(ctx: ClientContext, query: string, options?: SearchOptions): Promise<SearchHits>;
|
|
33
|
+
/**
|
|
34
|
+
* A running crawl.
|
|
35
|
+
*
|
|
36
|
+
* Iterating it streams pages as they are scraped. Results are never batched: a
|
|
37
|
+
* caller sees page one while page one hundred is still in the frontier, which
|
|
38
|
+
* is the only delivery model that makes a hundred-page crawl usable.
|
|
39
|
+
*/
|
|
40
|
+
export declare class CrawlHandle implements AsyncIterable<CrawlPage> {
|
|
41
|
+
#private;
|
|
42
|
+
readonly id: string;
|
|
43
|
+
readonly url: string;
|
|
44
|
+
readonly createdAt: string;
|
|
45
|
+
constructor(job: CrawlJob, source: AsyncIterator<unknown>);
|
|
46
|
+
get status(): CrawlJob['status'];
|
|
47
|
+
/** Pages completed so far. Crawls stream, so this moves as you iterate. */
|
|
48
|
+
get completed(): number;
|
|
49
|
+
/** Pages the frontier knows about — an estimate until the crawl ends. */
|
|
50
|
+
get discovered(): number;
|
|
51
|
+
get usage(): Usage;
|
|
52
|
+
[Symbol.asyncIterator](): AsyncIterator<CrawlPage>;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Start a crawl.
|
|
56
|
+
*
|
|
57
|
+
* One capability, one call. The registry row declares the response framing, so
|
|
58
|
+
* this reads a stream because `crawl.framing` says `'ndjson'` and not because
|
|
59
|
+
* anyone here decided crawls stream: a header frame naming the job, then one
|
|
60
|
+
* `CrawlPage` per line as the frontier produces them. Awaiting `crawl()`
|
|
61
|
+
* resolves on the header, so a caller has the id and can begin iterating
|
|
62
|
+
* without a second round trip.
|
|
63
|
+
*
|
|
64
|
+
* If the row is ever declared `'json'`, this makes one plain request and the
|
|
65
|
+
* handle yields nothing — no invented pages, and no polling of a route the
|
|
66
|
+
* registry does not declare.
|
|
67
|
+
*/
|
|
68
|
+
export declare function crawl(ctx: ClientContext, url: string, options?: CrawlOptions): Promise<CrawlHandle>;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Secrets — a credential an agent uses and never sees.
|
|
3
|
+
*
|
|
4
|
+
* **There is no `get` and there never will be.** A read route would be a route
|
|
5
|
+
* somebody is eventually asked to make return the value, and the reason this
|
|
6
|
+
* store is worth having is that no such route exists to widen. `list` answers
|
|
7
|
+
* everything a human needs in order to pick a handle: the handle, the kind, and
|
|
8
|
+
* a card's last four digits.
|
|
9
|
+
*
|
|
10
|
+
* A run reaches a secret only by naming its HANDLE in the submit
|
|
11
|
+
* (`AgentRequest.secrets`), and the substitution happens below the model, at
|
|
12
|
+
* the moment the keystrokes reach the page. `packages/core/src/secrets.ts`
|
|
13
|
+
* states the rule; this class is the management half of it, for a human.
|
|
14
|
+
*/
|
|
15
|
+
import type { Secret, SecretCreateRequest, SecretQuery } from './_deps/core/index.js';
|
|
16
|
+
import { type ClientContext, type Deleted, type Paged, type Result } from './result.js';
|
|
17
|
+
export declare class Secrets {
|
|
18
|
+
#private;
|
|
19
|
+
constructor(ctx: ClientContext);
|
|
20
|
+
/**
|
|
21
|
+
* Store one credential and get back the row WITHOUT its value.
|
|
22
|
+
*
|
|
23
|
+
* The response is the only receipt that the value was stored, and `hint` is
|
|
24
|
+
* the only part of it that says anything about what was stored.
|
|
25
|
+
*/
|
|
26
|
+
create(request: SecretCreateRequest): Promise<Result<Secret>>;
|
|
27
|
+
list(query?: SecretQuery): Promise<Result<Paged<Secret>>>;
|
|
28
|
+
/**
|
|
29
|
+
* Delete a credential and the record of which runs used it.
|
|
30
|
+
*
|
|
31
|
+
* Takes effect on the next fill rather than at the end of the run: a run
|
|
32
|
+
* holding a grant finds nothing to resolve and refuses by name, which is
|
|
33
|
+
* what "delete my card now" has to mean.
|
|
34
|
+
*/
|
|
35
|
+
delete(id: string): Promise<Result<Deleted>>;
|
|
36
|
+
}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sessions and replay — the differentiator, with a first-class surface rather
|
|
3
|
+
* than a URL buried in an agent result.
|
|
4
|
+
*
|
|
5
|
+
* Every run has a replay. A customer disputing a charge gets to watch exactly
|
|
6
|
+
* what their money bought, step by step, which turns the worst support
|
|
7
|
+
* conversation in this industry into a link.
|
|
8
|
+
*
|
|
9
|
+
* Note there is no `identity` filter, deliberately, and one is refused by name
|
|
10
|
+
* rather than ignored. The equivalent internal query — "every brave block this
|
|
11
|
+
* week" — lives on the operator side, where the identity loop is run. Customers
|
|
12
|
+
* see outcomes; we see which identity produced them.
|
|
13
|
+
*/
|
|
14
|
+
import { type BilledResources, type ResolvedExit, type SessionExport, type SessionOutcome, type SessionQuery, type SessionStatus, type SessionSummary, type Timestamp, type Usage } from './_deps/core/index.js';
|
|
15
|
+
import type { ReplayEvent } from './_deps/replay/events.js';
|
|
16
|
+
import { Browser } from './browser.js';
|
|
17
|
+
import type { HandsFactory, RawPage } from './hands.js';
|
|
18
|
+
import type { OutcrawlPage } from './page.js';
|
|
19
|
+
import { type ClientContext, type Paged, type Result } from './result.js';
|
|
20
|
+
/**
|
|
21
|
+
* A session filter the API cannot honour. Named, not ignored: a silently
|
|
22
|
+
* dropped filter returns the wrong sessions and looks like it worked.
|
|
23
|
+
*/
|
|
24
|
+
export declare class UnsupportedSessionFilterError extends Error {
|
|
25
|
+
readonly name = "UnsupportedSessionFilterError";
|
|
26
|
+
readonly filter: string;
|
|
27
|
+
constructor(filter: string, why: string);
|
|
28
|
+
}
|
|
29
|
+
export declare function assertSupportedFilter(query: object): void;
|
|
30
|
+
/**
|
|
31
|
+
* Interactive control of a running session.
|
|
32
|
+
*
|
|
33
|
+
* The practical answer to an agent stuck on a consent dialog at 2am, and the
|
|
34
|
+
* thing screenshots and video cannot do. Input goes through the same human path
|
|
35
|
+
* as everywhere else — taking over must not change how the session looks.
|
|
36
|
+
*/
|
|
37
|
+
export declare class SessionControl {
|
|
38
|
+
#private;
|
|
39
|
+
constructor(page: OutcrawlPage<RawPage>, browser: Browser);
|
|
40
|
+
/** Playwright's surface, untouched and recorded, exactly as on a page. */
|
|
41
|
+
get raw(): RawPage;
|
|
42
|
+
click(selector: string): Promise<void>;
|
|
43
|
+
fill(selector: string, value: string): Promise<void>;
|
|
44
|
+
type(selector: string, text: string): Promise<void>;
|
|
45
|
+
hover(selector: string): Promise<void>;
|
|
46
|
+
press(selector: string, key: string): Promise<void>;
|
|
47
|
+
selectOption(selector: string, values: readonly string[]): Promise<void>;
|
|
48
|
+
/** Hand the session back to the agent. */
|
|
49
|
+
release(): Promise<void>;
|
|
50
|
+
}
|
|
51
|
+
export interface SessionDeps extends ClientContext {
|
|
52
|
+
hands: HandsFactory;
|
|
53
|
+
}
|
|
54
|
+
/** One session, and everything you can do with it. */
|
|
55
|
+
export declare class Session {
|
|
56
|
+
#private;
|
|
57
|
+
/** The record as the API returned it. */
|
|
58
|
+
readonly summary: SessionSummary;
|
|
59
|
+
constructor(summary: SessionSummary, ctx: SessionDeps);
|
|
60
|
+
get id(): string;
|
|
61
|
+
get status(): SessionStatus;
|
|
62
|
+
get outcome(): SessionOutcome | undefined;
|
|
63
|
+
get task(): string | undefined;
|
|
64
|
+
get profileId(): string | undefined;
|
|
65
|
+
/** Where this session actually came out. Session-scoped, not profile-scoped. */
|
|
66
|
+
get exit(): ResolvedExit | undefined;
|
|
67
|
+
get startedAt(): Timestamp;
|
|
68
|
+
get endedAt(): Timestamp | undefined;
|
|
69
|
+
/** Wall clock, in milliseconds. */
|
|
70
|
+
get duration(): number;
|
|
71
|
+
get pages(): number;
|
|
72
|
+
/**
|
|
73
|
+
* Hosted player, or `undefined` when this run was not durably recorded.
|
|
74
|
+
*
|
|
75
|
+
* `undefined` and not a thrown error or an empty string: a host with no
|
|
76
|
+
* durable replay storage records into memory and frees it on exit, so it
|
|
77
|
+
* publishes no link at all rather than one that 404s for the whole retention
|
|
78
|
+
* window. `SessionSummary.replayUrl` is optional for that reason and this
|
|
79
|
+
* getter reports it faithfully — an empty string would be a dead link with
|
|
80
|
+
* extra steps.
|
|
81
|
+
*/
|
|
82
|
+
get replayUrl(): string | undefined;
|
|
83
|
+
get usage(): Usage;
|
|
84
|
+
get billed(): BilledResources | undefined;
|
|
85
|
+
/**
|
|
86
|
+
* Export the session as a portable bundle that plays offline.
|
|
87
|
+
*
|
|
88
|
+
* With a destination it is written there; without one you get the descriptor
|
|
89
|
+
* and can fetch the URL yourself.
|
|
90
|
+
*/
|
|
91
|
+
export(destination?: string): Promise<Result<SessionExport>>;
|
|
92
|
+
/**
|
|
93
|
+
* Stream events while the session is still running.
|
|
94
|
+
*
|
|
95
|
+
* The same registry row as `get`, switched to a stream by the field that row
|
|
96
|
+
* declares in `streamsWhen`. Same session, followed rather than snapshotted —
|
|
97
|
+
* one row, not a second one only the SDK would carry, and not a query
|
|
98
|
+
* parameter this package invented.
|
|
99
|
+
*
|
|
100
|
+
* Events use the `ReplayEvent` vocabulary of the recorded format, so a live
|
|
101
|
+
* consumer and a replay consumer are the same code.
|
|
102
|
+
*/
|
|
103
|
+
live(signal?: AbortSignal): AsyncIterable<ReplayEvent>;
|
|
104
|
+
/**
|
|
105
|
+
* Take the wheel on a running session. Attaches to the existing page rather
|
|
106
|
+
* than opening a new one — the point is to unstick the run that is already
|
|
107
|
+
* there.
|
|
108
|
+
*/
|
|
109
|
+
takeControl(): Promise<SessionControl>;
|
|
110
|
+
}
|
|
111
|
+
export declare class Sessions {
|
|
112
|
+
#private;
|
|
113
|
+
constructor(ctx: SessionDeps);
|
|
114
|
+
get(id: string): Promise<Session>;
|
|
115
|
+
/**
|
|
116
|
+
* The two queries that make replay pay for itself:
|
|
117
|
+
* `{ outcome: 'blocked', since: '7d' }` and
|
|
118
|
+
* `{ status: 'partial', task: '*checkout*' }`. Twenty replays to watch rather
|
|
119
|
+
* than twenty log lines to guess from.
|
|
120
|
+
*/
|
|
121
|
+
list(query?: SessionQuery): Promise<Paged<SessionSummary>>;
|
|
122
|
+
/** Export by id, without fetching the session first. */
|
|
123
|
+
export(id: string, destination?: string): Promise<Result<SessionExport>>;
|
|
124
|
+
}
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The transport seam.
|
|
3
|
+
*
|
|
4
|
+
* Everything the SDK does that leaves the process goes through {@link Transport}.
|
|
5
|
+
* That is not an abstraction for its own sake: it is the reason the whole client
|
|
6
|
+
* — including the Playwright page integration, `observe`/`act`, session live
|
|
7
|
+
* streams and interactive control — is testable with no network and no browser.
|
|
8
|
+
*
|
|
9
|
+
* There are two channels here and they are different in kind, exactly as the
|
|
10
|
+
* product surface says:
|
|
11
|
+
*
|
|
12
|
+
* - `request` / `stream` / `download` speak to the REST API, and every one of
|
|
13
|
+
* those calls is addressed by a **capability name** from `@outcrawl/core`.
|
|
14
|
+
* The method and path are read from the registry, never written out here, so
|
|
15
|
+
* a route cannot drift from the registry row that declares it.
|
|
16
|
+
* - `connect` opens a live CDP session. The live `page` surface only exists in
|
|
17
|
+
* the SDK because it needs a CDP connection in the caller's own process; API,
|
|
18
|
+
* MCP and CLI reach the same capabilities through `agent` and `scrape`, which
|
|
19
|
+
* run the loop on our side.
|
|
20
|
+
*/
|
|
21
|
+
import { type CapabilityName, type ExitTarget, type HttpMethod, type JsonSchema, type ObservedAction, type Timestamp, type Usage } from './_deps/core/index.js';
|
|
22
|
+
import type { RawPage, WheelPath } from './hands.js';
|
|
23
|
+
/**
|
|
24
|
+
* One call to one capability. `capability` is the registry key; `method` and
|
|
25
|
+
* `path` are read off the registry row by {@link resolveRequest} rather than
|
|
26
|
+
* supplied by the caller, so a fake transport in a test sees exactly the method
|
|
27
|
+
* and path the registry declares.
|
|
28
|
+
*/
|
|
29
|
+
export interface RequestSpec {
|
|
30
|
+
capability: CapabilityName;
|
|
31
|
+
method: HttpMethod;
|
|
32
|
+
/** Registry path with every `{param}` substituted. */
|
|
33
|
+
path: string;
|
|
34
|
+
query?: Readonly<Record<string, string>>;
|
|
35
|
+
body?: unknown;
|
|
36
|
+
signal?: AbortSignal;
|
|
37
|
+
}
|
|
38
|
+
export interface RequestInit_ {
|
|
39
|
+
params?: Readonly<Record<string, string>>;
|
|
40
|
+
query?: Readonly<Record<string, string>>;
|
|
41
|
+
body?: unknown;
|
|
42
|
+
signal?: AbortSignal;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Substitute `{id}`-style path parameters and build the {@link RequestSpec} for
|
|
46
|
+
* a capability. The registry is the only source of the method and the path
|
|
47
|
+
* template — nothing in this package writes a URL out by hand.
|
|
48
|
+
*
|
|
49
|
+
* Refuses a capability nothing serves, before building anything. This is the
|
|
50
|
+
* chokepoint rather than the individual client methods because it is the one
|
|
51
|
+
* place every REST call in the package already passes through: a method added
|
|
52
|
+
* later cannot reach an unserved route by forgetting to ask, and neither can a
|
|
53
|
+
* caller using this function directly from the `@outcrawl/sdk/transport`
|
|
54
|
+
* export.
|
|
55
|
+
*/
|
|
56
|
+
export declare function resolveRequest(capability: CapabilityName, init?: RequestInit_): RequestSpec;
|
|
57
|
+
/** True when this row's default response framing is a stream. */
|
|
58
|
+
export declare function alwaysStreams(capability: CapabilityName): boolean;
|
|
59
|
+
/**
|
|
60
|
+
* The request field that switches a row to `ndjson`.
|
|
61
|
+
*
|
|
62
|
+
* Read from the registry rather than written out, because the last time a
|
|
63
|
+
* surface wrote one out it invented `live` six rows ahead of the registry that
|
|
64
|
+
* was supposed to be the single source of it.
|
|
65
|
+
*/
|
|
66
|
+
export declare function streamSwitch(capability: CapabilityName): string;
|
|
67
|
+
/**
|
|
68
|
+
* What the transport is told when a browser is opened. Two fields, because a
|
|
69
|
+
* profile and a geography are the only two things a caller chooses. Identity,
|
|
70
|
+
* seed, fingerprint, timezone, languages and WebRTC address are minted by us;
|
|
71
|
+
* `assertNoIdentityOverride` in `browser.ts` refuses anything else by name
|
|
72
|
+
* before a socket is opened.
|
|
73
|
+
*/
|
|
74
|
+
export interface ConnectSpec {
|
|
75
|
+
profile?: string;
|
|
76
|
+
exit?: ExitTarget;
|
|
77
|
+
/**
|
|
78
|
+
* The caller declined this session's recording, with `record: false` on
|
|
79
|
+
* `oc.browser()`.
|
|
80
|
+
*
|
|
81
|
+
* `declineRecording: true` and not `record: false`, because only the refusal
|
|
82
|
+
* ever reaches the transport: absence already means recorded, so there is no
|
|
83
|
+
* second value for this field to carry and no way to spell the default.
|
|
84
|
+
* `HttpTransport.connect` turns it into `?record=false` on the allocation
|
|
85
|
+
* request; the router normalises that and signs the decision into the ticket,
|
|
86
|
+
* so nothing between here and the machine can strip it.
|
|
87
|
+
*/
|
|
88
|
+
declineRecording?: true;
|
|
89
|
+
/** Attach to a session that is already running, for `session.takeControl()`. */
|
|
90
|
+
attachTo?: string;
|
|
91
|
+
signal?: AbortSignal;
|
|
92
|
+
}
|
|
93
|
+
/** Reviewable actions plus the usage the resolution cost. */
|
|
94
|
+
export interface ObserveResponse {
|
|
95
|
+
actions: ObservedAction[];
|
|
96
|
+
usage: Usage;
|
|
97
|
+
}
|
|
98
|
+
export interface ExtractResponse {
|
|
99
|
+
data: unknown;
|
|
100
|
+
usage: Usage;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* A live CDP session. `observe` and `extract` are session-scoped RPCs on our own
|
|
104
|
+
* endpoint rather than REST capabilities — that is the one documented
|
|
105
|
+
* difference in kind between the SDK and the other three surfaces.
|
|
106
|
+
*/
|
|
107
|
+
export interface Connection {
|
|
108
|
+
readonly sessionId: string;
|
|
109
|
+
/**
|
|
110
|
+
* Seeds the human-input timing so one identity keeps its typing pace and
|
|
111
|
+
* click character across sessions — real typing pace varies 3.6x between
|
|
112
|
+
* people, and an identity that returns at a different speed every visit is
|
|
113
|
+
* describing someone who is not the same person.
|
|
114
|
+
*
|
|
115
|
+
* Derived locally from the profile id the caller already named, never from
|
|
116
|
+
* the wire. A persona field on the allocation response would make identity a
|
|
117
|
+
* de facto part of the public API, and the next request after that is a
|
|
118
|
+
* customer asking to choose one. An ephemeral browser has no profile, so it
|
|
119
|
+
* falls back to the session id and its persona lasts exactly one session,
|
|
120
|
+
* which is all an ephemeral identity is.
|
|
121
|
+
*/
|
|
122
|
+
readonly persona: string;
|
|
123
|
+
/** When the allocation ticket stops being redeemable. */
|
|
124
|
+
readonly expiresAt: string;
|
|
125
|
+
newPage(): Promise<RawPage>;
|
|
126
|
+
/**
|
|
127
|
+
* The browser's batched wheel entry point, `Outcrawl.dispatchWheelPath`.
|
|
128
|
+
*
|
|
129
|
+
* On the {@link Connection} rather than reached for through a raw CDP handle,
|
|
130
|
+
* because it is the one input path the SDK cannot express through Playwright:
|
|
131
|
+
* `Input.dispatchMouseWheel` has no parameter for a wheel event's phase, so it
|
|
132
|
+
* marks every event `began` and a page sees one cancelable wheel event and one
|
|
133
|
+
* `scrollend` per EVENT where a real device gives one of each per GESTURE.
|
|
134
|
+
* Measured on the shipped binary at 1600px: 3 of 108 cancelable and 3
|
|
135
|
+
* `scrollend` for three gestures through here, against 147 of 147 and 98 for
|
|
136
|
+
* the same library, profile and seed over `Input.dispatchMouseWheel`.
|
|
137
|
+
*
|
|
138
|
+
* Rejects while the session holds `Outcrawl.lock`, by design — the lock drops
|
|
139
|
+
* every input event before the renderer sees it, so a command that reported
|
|
140
|
+
* success would be lying.
|
|
141
|
+
*/
|
|
142
|
+
dispatchWheelPath(path: WheelPath): Promise<unknown>;
|
|
143
|
+
/** Resolve an instruction to candidate actions. Executes none of them. */
|
|
144
|
+
observe(instruction: string, options?: {
|
|
145
|
+
cache?: boolean;
|
|
146
|
+
}): Promise<ObserveResponse>;
|
|
147
|
+
extract(instruction: string, schema?: JsonSchema): Promise<ExtractResponse>;
|
|
148
|
+
close(): Promise<void>;
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* A raw-input call, recorded against the session.
|
|
152
|
+
*
|
|
153
|
+
* `page.raw.click()` is the seam where a customer can accidentally undo the
|
|
154
|
+
* product: the click lands at Playwright's exact bounding-box centre with zero
|
|
155
|
+
* variance rather than going through `agenthands`. It stays available — setup
|
|
156
|
+
* steps where speed matters are a real need — but a run that used it is flagged
|
|
157
|
+
* rather than silently different, so a session that later gets blocked is not a
|
|
158
|
+
* mystery to whoever reads the replay.
|
|
159
|
+
*/
|
|
160
|
+
export interface RawInputMark {
|
|
161
|
+
kind: 'raw-input';
|
|
162
|
+
/** The Playwright method that was reached for, e.g. `click`. */
|
|
163
|
+
method: string;
|
|
164
|
+
/** The selector, or the namespace name for `mouse` / `keyboard`. */
|
|
165
|
+
target?: string;
|
|
166
|
+
at: Timestamp;
|
|
167
|
+
}
|
|
168
|
+
export interface Transport {
|
|
169
|
+
request<T>(spec: RequestSpec): Promise<T>;
|
|
170
|
+
/** Newline-delimited JSON: crawl pages, agent steps, live session events. */
|
|
171
|
+
stream(spec: RequestSpec): AsyncIterable<unknown>;
|
|
172
|
+
/** Fetch a bundle the API handed us a URL for, e.g. a session export. */
|
|
173
|
+
download(url: string, signal?: AbortSignal): Promise<Uint8Array>;
|
|
174
|
+
connect(spec: ConnectSpec): Promise<Connection>;
|
|
175
|
+
/**
|
|
176
|
+
* Annotate a live session out of band. Fire-and-forget by design: a mark must
|
|
177
|
+
* never be able to fail, slow or reorder a customer's click.
|
|
178
|
+
*/
|
|
179
|
+
mark(sessionId: string, mark: RawInputMark): void;
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* An error body the client could not turn into a typed error — an unknown code
|
|
183
|
+
* from a newer server, or a response that was not an error envelope at all.
|
|
184
|
+
*
|
|
185
|
+
* Lenient on purpose. A client that rejected a body because the code is newer
|
|
186
|
+
* than the client would turn a 404 into a crash, during an incident, which is
|
|
187
|
+
* exactly when a new code is most likely to be in flight.
|
|
188
|
+
*/
|
|
189
|
+
export declare class OutcrawlApiError extends Error {
|
|
190
|
+
readonly name = "OutcrawlApiError";
|
|
191
|
+
readonly status: number;
|
|
192
|
+
readonly body: unknown;
|
|
193
|
+
/** The wire code, when there was one. Unknown to this client, not invalid. */
|
|
194
|
+
readonly code: string | undefined;
|
|
195
|
+
constructor(status: number, body: unknown, detail?: string);
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* Rebuild a typed error from the wire.
|
|
199
|
+
*
|
|
200
|
+
* The router returns core's `SerializedOutcrawlError` bare — `{ code, message,
|
|
201
|
+
* details }` with an HTTP status, no wrapper. Reconstructing the class rather
|
|
202
|
+
* than surfacing a generic HTTP failure is what makes
|
|
203
|
+
* `catch (e) { if (e instanceof ProfileInUseError) e.heldUntil }` work in a
|
|
204
|
+
* customer's own code, which is the entire value of the leased-profile
|
|
205
|
+
* contract: a retry that knows when to retry.
|
|
206
|
+
*
|
|
207
|
+
* Parsed as core's `WireError`, whose `code` is an open string — strict in what
|
|
208
|
+
* we emit, lenient in what we accept. A code this client does not know falls
|
|
209
|
+
* through to {@link OutcrawlApiError} carrying the code and body, rather than
|
|
210
|
+
* failing to parse.
|
|
211
|
+
*/
|
|
212
|
+
export declare function errorFromWire(status: number, body: unknown): Error;
|
|
213
|
+
/** The slice of Playwright's `Browser` the SDK drives. */
|
|
214
|
+
export interface CdpBrowser {
|
|
215
|
+
contexts(): CdpBrowserContext[];
|
|
216
|
+
newContext(): Promise<CdpBrowserContext>;
|
|
217
|
+
close(): Promise<void>;
|
|
218
|
+
}
|
|
219
|
+
export interface CdpBrowserContext {
|
|
220
|
+
pages(): RawPage[];
|
|
221
|
+
newPage(): Promise<RawPage>;
|
|
222
|
+
newCDPSession(page: RawPage): Promise<CdpSession>;
|
|
223
|
+
}
|
|
224
|
+
export interface CdpSession {
|
|
225
|
+
send(method: string, params?: Record<string, unknown>): Promise<unknown>;
|
|
226
|
+
}
|
|
227
|
+
export interface HttpTransportOptions {
|
|
228
|
+
apiKey: string;
|
|
229
|
+
/** Defaults to `https://outcrawl.ai`. */
|
|
230
|
+
baseUrl?: string;
|
|
231
|
+
/** Injected so the whole REST surface is exercisable without a network. */
|
|
232
|
+
fetch?: typeof globalThis.fetch;
|
|
233
|
+
/**
|
|
234
|
+
* Opens a CDP connection to a WebSocket endpoint. Defaults to Playwright's
|
|
235
|
+
* `chromium.connectOverCDP`, imported lazily so a caller who never opens a
|
|
236
|
+
* browser never loads Playwright — the SDK is optional, and so is its browser
|
|
237
|
+
* half.
|
|
238
|
+
*/
|
|
239
|
+
connectOverCDP?: (wsEndpoint: string) => Promise<CdpBrowser>;
|
|
240
|
+
}
|
|
241
|
+
export declare class HttpTransport implements Transport {
|
|
242
|
+
#private;
|
|
243
|
+
readonly baseUrl: string;
|
|
244
|
+
constructor(options: HttpTransportOptions);
|
|
245
|
+
request<T>(spec: RequestSpec): Promise<T>;
|
|
246
|
+
stream(spec: RequestSpec): AsyncIterable<unknown>;
|
|
247
|
+
download(url: string, signal?: AbortSignal): Promise<Uint8Array>;
|
|
248
|
+
connect(spec: ConnectSpec): Promise<Connection>;
|
|
249
|
+
mark(sessionId: string, mark: RawInputMark): void;
|
|
250
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@outcrawl/sdk",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"description": "Outcrawl TypeScript client — scrape, crawl, search and drive real stealth browsers over CDP, on one credit balance.",
|
|
6
|
+
"keywords": [
|
|
7
|
+
"outcrawl",
|
|
8
|
+
"scraping",
|
|
9
|
+
"crawler",
|
|
10
|
+
"web-search",
|
|
11
|
+
"browser-automation",
|
|
12
|
+
"playwright",
|
|
13
|
+
"cdp",
|
|
14
|
+
"stealth",
|
|
15
|
+
"agent"
|
|
16
|
+
],
|
|
17
|
+
"homepage": "https://outcrawl.ai",
|
|
18
|
+
"repository": {
|
|
19
|
+
"type": "git",
|
|
20
|
+
"url": "git+https://github.com/aeonmindai/outcrawl-browser.git",
|
|
21
|
+
"directory": "packages/sdk"
|
|
22
|
+
},
|
|
23
|
+
"bugs": {
|
|
24
|
+
"url": "https://github.com/aeonmindai/outcrawl-browser/issues"
|
|
25
|
+
},
|
|
26
|
+
"main": "./dist/index.js",
|
|
27
|
+
"module": "./dist/index.js",
|
|
28
|
+
"types": "./dist/types/index.d.ts",
|
|
29
|
+
"exports": {
|
|
30
|
+
".": {
|
|
31
|
+
"types": "./dist/types/index.d.ts",
|
|
32
|
+
"import": "./dist/index.js",
|
|
33
|
+
"default": "./dist/index.js"
|
|
34
|
+
},
|
|
35
|
+
"./package.json": "./package.json"
|
|
36
|
+
},
|
|
37
|
+
"engines": {
|
|
38
|
+
"node": ">=20"
|
|
39
|
+
},
|
|
40
|
+
"publishConfig": {
|
|
41
|
+
"access": "public"
|
|
42
|
+
},
|
|
43
|
+
"files": [
|
|
44
|
+
"dist/index.js",
|
|
45
|
+
"dist/types"
|
|
46
|
+
],
|
|
47
|
+
"scripts": {
|
|
48
|
+
"build": "bun ./scripts/build.ts",
|
|
49
|
+
"prepack": "bun ./scripts/build.ts",
|
|
50
|
+
"test": "bun test ./test"
|
|
51
|
+
},
|
|
52
|
+
"peerDependencies": {
|
|
53
|
+
"agenthands": ">=1.1.1",
|
|
54
|
+
"playwright": "^1.56.0"
|
|
55
|
+
},
|
|
56
|
+
"peerDependenciesMeta": {
|
|
57
|
+
"agenthands": {
|
|
58
|
+
"optional": true
|
|
59
|
+
},
|
|
60
|
+
"playwright": {
|
|
61
|
+
"optional": true
|
|
62
|
+
}
|
|
63
|
+
},
|
|
64
|
+
"devDependencies": {
|
|
65
|
+
"@outcrawl/core": "workspace:*",
|
|
66
|
+
"@outcrawl/integrations": "workspace:*",
|
|
67
|
+
"@outcrawl/proxy": "workspace:*",
|
|
68
|
+
"@outcrawl/replay": "workspace:*"
|
|
69
|
+
}
|
|
70
|
+
}
|