@oya-ai/browser 1.0.86 → 1.0.88
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -1
- package/dist/index.cjs +72 -9
- package/dist/index.d.cts +107 -11
- package/dist/index.d.ts +107 -11
- package/dist/index.js +69 -8
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -150,6 +150,28 @@ Use `{{name}}` references in prompts instead of interpolating values into the pr
|
|
|
150
150
|
|
|
151
151
|
For prompt runs, pass credentials in `secrets`. For replay, pass all variables in `data`: the saved playbook tracks which variables are secret, including during healing. Placeholder-based inputs remain variables in the saved flow. This is not a blanket redaction guarantee for page content, screenshots, agent replies, or application logs; inspect exported code before sharing it.
|
|
152
152
|
|
|
153
|
+
### Files
|
|
154
|
+
|
|
155
|
+
`file()` puts a file in `data`. The agent attaches it with its upload tool: hand it the element id of whatever you can see — the "Choose file" button, the drop zone, the field itself — and the real `<input type="file">` is found from there, including the hidden ones most upload widgets use.
|
|
156
|
+
|
|
157
|
+
```js
|
|
158
|
+
import { Oya, file } from "@oya-ai/browser";
|
|
159
|
+
|
|
160
|
+
await browser.ask("Attach my resume to the application and submit it", {
|
|
161
|
+
data: { name: "Ada Lovelace", resume: await file("./cv.pdf") },
|
|
162
|
+
});
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
A string argument is a path on disk (Node only); a `Blob`, a `File`, or a `Uint8Array` works anywhere. `name` sets the filename the site sees and `type` overrides the MIME guessed from the extension. The ceiling is 10MB per file, and the bytes travel inline with the run — nothing is stored server-side after it ends.
|
|
166
|
+
|
|
167
|
+
Files work in `data` for `ask()`, `submit()`, and `play()`; `secrets` rejects them, because a file is never typed through a placeholder. A run recorded with `toPlaybook()` keeps the upload as a variable, so the replay takes a different file:
|
|
168
|
+
|
|
169
|
+
```js
|
|
170
|
+
await browser.play("job-application", { name: "Ada Lovelace", resume: await file("./other.pdf") });
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
The generated Playwright module calls `setInputFiles`, where the same variable is a plain path rather than a `file()` value.
|
|
174
|
+
|
|
153
175
|
### Review a repaired playbook
|
|
154
176
|
|
|
155
177
|
Replay normally runs recorded steps without an LLM. With `autoHeal: true` (the default), a broken step can hand over to the agent, which saves a repair as `<name>:draft`. Promotion replaces the saved playbook with that draft.
|
|
@@ -179,12 +201,28 @@ const modelKey = process.env.GEMINI_API_KEY;
|
|
|
179
201
|
if (!modelKey) throw new Error("Set GEMINI_API_KEY first.");
|
|
180
202
|
const oya = new Oya();
|
|
181
203
|
await oya.config.set({
|
|
182
|
-
llm_provider: "gemini", // "openai" | "anthropic" | "gemini" | "vertex"
|
|
204
|
+
llm_provider: "gemini", // LlmProvider: "openai" | "anthropic" | "gemini" | "vertex"
|
|
183
205
|
openai_api_key: modelKey, // Shared field name for every supported provider.
|
|
184
206
|
// chat_model: process.env.OYA_CHAT_MODEL, // Optional provider model override.
|
|
185
207
|
});
|
|
186
208
|
```
|
|
187
209
|
|
|
210
|
+
`config.set` takes `ConfigUpdate` and `config.get()` returns `Config`, so an editor offers the valid providers and a typo fails to compile rather than silently falling back:
|
|
211
|
+
|
|
212
|
+
```ts
|
|
213
|
+
import { Oya, type LlmProvider } from "@oya-ai/browser";
|
|
214
|
+
|
|
215
|
+
await oya.config.set({ llm_provider: "vertx" });
|
|
216
|
+
// ~~~~~~~ Type '"vertx"' is not assignable to type
|
|
217
|
+
// 'LlmProvider'. Did you mean '"vertex"'?
|
|
218
|
+
|
|
219
|
+
const provider: LlmProvider = "vertex"; // for your own config plumbing
|
|
220
|
+
const { effective } = await oya.config.get();
|
|
221
|
+
console.log(effective.baseUrl, effective.model, effective.hasLlmKey);
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
`browser_provider` is typed as `Provider` and `captcha_solver` as `CaptchaSolver` the same way. The server enforces the same sets, so a non-TypeScript caller gets a 400 listing the valid values instead of a silent fallback. Pass `null` to clear a field.
|
|
225
|
+
|
|
188
226
|
### Gemini Enterprise (ex-Vertex AI)
|
|
189
227
|
|
|
190
228
|
`llm_provider: "vertex"` targets express mode, whose API keys work against a global endpoint with no GCP project or location:
|
package/dist/index.cjs
CHANGED
|
@@ -21,10 +21,12 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
21
21
|
var index_exports = {};
|
|
22
22
|
__export(index_exports, {
|
|
23
23
|
Browser: () => Browser,
|
|
24
|
+
MAX_FILE_BYTES: () => MAX_FILE_BYTES,
|
|
24
25
|
Oya: () => Oya,
|
|
25
26
|
OyaError: () => OyaError,
|
|
26
27
|
Run: () => Run,
|
|
27
|
-
default: () => index_default
|
|
28
|
+
default: () => index_default,
|
|
29
|
+
file: () => file
|
|
28
30
|
});
|
|
29
31
|
module.exports = __toCommonJS(index_exports);
|
|
30
32
|
|
|
@@ -203,9 +205,10 @@ var Browser = class {
|
|
|
203
205
|
return res.text;
|
|
204
206
|
}
|
|
205
207
|
/**
|
|
206
|
-
* Save the last `ask()` on this browser as a named playbook.
|
|
207
|
-
*
|
|
208
|
-
*
|
|
208
|
+
* Save the last `ask()` on this browser as a named playbook. Every value that was
|
|
209
|
+
* typed, picked or clicked becomes a variable, with what the run used kept in
|
|
210
|
+
* `defaults`, so `play()` with nothing repeats the run and any one value can be
|
|
211
|
+
* swapped. `code` is the same flow as a Playwright module, to read or run yourself.
|
|
209
212
|
*/
|
|
210
213
|
toPlaybook(name) {
|
|
211
214
|
return this.http.request("POST", `/api/browsers/${this.id}/playbooks`, { name }, 12e4);
|
|
@@ -376,6 +379,63 @@ var Run = class {
|
|
|
376
379
|
}
|
|
377
380
|
};
|
|
378
381
|
|
|
382
|
+
// src/file.ts
|
|
383
|
+
var MAX_FILE_BYTES = 10 * 1024 * 1024;
|
|
384
|
+
var NODE_FS = "node:fs/promises";
|
|
385
|
+
var MIME = {
|
|
386
|
+
pdf: "application/pdf",
|
|
387
|
+
png: "image/png",
|
|
388
|
+
jpg: "image/jpeg",
|
|
389
|
+
jpeg: "image/jpeg",
|
|
390
|
+
gif: "image/gif",
|
|
391
|
+
webp: "image/webp",
|
|
392
|
+
svg: "image/svg+xml",
|
|
393
|
+
heic: "image/heic",
|
|
394
|
+
txt: "text/plain",
|
|
395
|
+
csv: "text/csv",
|
|
396
|
+
json: "application/json",
|
|
397
|
+
xml: "application/xml",
|
|
398
|
+
html: "text/html",
|
|
399
|
+
doc: "application/msword",
|
|
400
|
+
docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
401
|
+
xls: "application/vnd.ms-excel",
|
|
402
|
+
xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
403
|
+
ppt: "application/vnd.ms-powerpoint",
|
|
404
|
+
pptx: "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
|
405
|
+
zip: "application/zip"
|
|
406
|
+
};
|
|
407
|
+
var base64 = (bytes) => {
|
|
408
|
+
const buffer = globalThis.Buffer;
|
|
409
|
+
if (buffer) return buffer.from(bytes).toString("base64");
|
|
410
|
+
let binary = "";
|
|
411
|
+
for (let i = 0; i < bytes.length; i += 8192) binary += String.fromCharCode(...bytes.subarray(i, i + 8192));
|
|
412
|
+
return btoa(binary);
|
|
413
|
+
};
|
|
414
|
+
async function file(source, options = {}) {
|
|
415
|
+
let bytes;
|
|
416
|
+
let name = options.name;
|
|
417
|
+
if (typeof source === "string") {
|
|
418
|
+
const fs = await import(
|
|
419
|
+
/* webpackIgnore: true */
|
|
420
|
+
/* @vite-ignore */
|
|
421
|
+
NODE_FS
|
|
422
|
+
);
|
|
423
|
+
bytes = new Uint8Array(await fs.readFile(source));
|
|
424
|
+
name ||= source.split(/[\\/]/).pop() || "file";
|
|
425
|
+
} else if (source instanceof Uint8Array) {
|
|
426
|
+
bytes = source;
|
|
427
|
+
} else {
|
|
428
|
+
bytes = new Uint8Array(await source.arrayBuffer());
|
|
429
|
+
name ||= source.name;
|
|
430
|
+
}
|
|
431
|
+
name ||= "file";
|
|
432
|
+
if (bytes.length > MAX_FILE_BYTES) {
|
|
433
|
+
throw new Error(`${name} is ${Math.round(bytes.length / 1024 / 1024)}MB; the limit for a task file is ${MAX_FILE_BYTES / 1024 / 1024}MB.`);
|
|
434
|
+
}
|
|
435
|
+
const ext = name.includes(".") ? name.split(".").pop().toLowerCase() : "";
|
|
436
|
+
return { file: name, type: options.type || MIME[ext] || "application/octet-stream", b64: base64(bytes) };
|
|
437
|
+
}
|
|
438
|
+
|
|
379
439
|
// src/index.ts
|
|
380
440
|
var DEFAULT_BASE_URL = "https://browser.getoya.ai";
|
|
381
441
|
var READY_POLL_MS = 2e3;
|
|
@@ -506,10 +566,11 @@ var Oya = class {
|
|
|
506
566
|
*
|
|
507
567
|
* Bring your own LLM key (it pays for its own tokens, so no hourly chat quota applies):
|
|
508
568
|
* await oya.config.set({ llm_provider: 'gemini', openai_api_key: process.env.GEMINI_API_KEY });
|
|
509
|
-
* `llm_provider` is
|
|
510
|
-
* its default model. 'vertex' is Gemini
|
|
511
|
-
* needs no GCP project; for a
|
|
512
|
-
* `.../endpoints/openapi` and pass an
|
|
569
|
+
* `llm_provider` is the {@link LlmProvider} union, so an editor offers the choices and a
|
|
570
|
+
* typo is a compile error; `chat_model` overrides its default model. 'vertex' is Gemini
|
|
571
|
+
* Enterprise (ex-Vertex AI) in express mode, which needs no GCP project; for a
|
|
572
|
+
* project-scoped endpoint, set `openai_base_url` to `.../endpoints/openapi` and pass an
|
|
573
|
+
* OAuth access token as `openai_api_key`. See {@link ConfigUpdate} for every field.
|
|
513
574
|
*/
|
|
514
575
|
config = {
|
|
515
576
|
get: () => this.http.request("GET", "/api/config"),
|
|
@@ -540,7 +601,9 @@ var index_default = Oya;
|
|
|
540
601
|
// Annotate the CommonJS export names for ESM import in node:
|
|
541
602
|
0 && (module.exports = {
|
|
542
603
|
Browser,
|
|
604
|
+
MAX_FILE_BYTES,
|
|
543
605
|
Oya,
|
|
544
606
|
OyaError,
|
|
545
|
-
Run
|
|
607
|
+
Run,
|
|
608
|
+
file
|
|
546
609
|
});
|
package/dist/index.d.cts
CHANGED
|
@@ -9,6 +9,60 @@ declare class Http {
|
|
|
9
9
|
}
|
|
10
10
|
|
|
11
11
|
/** Everything the API returns or accepts, in one place. */
|
|
12
|
+
/** Which model drives `ask()` and the chat API. */
|
|
13
|
+
type LlmProvider = 'openai' | 'anthropic'
|
|
14
|
+
/** Gemini via AI Studio. */
|
|
15
|
+
| 'gemini'
|
|
16
|
+
/** Gemini Enterprise, ex-Vertex AI. Express mode by default; set `openai_base_url` to a
|
|
17
|
+
* project-scoped `.../endpoints/openapi` endpoint to use an enterprise project. */
|
|
18
|
+
| 'vertex';
|
|
19
|
+
/** Empty disables solving. */
|
|
20
|
+
type CaptchaSolver = 'capsolver' | '2captcha' | '';
|
|
21
|
+
/**
|
|
22
|
+
* What a key can configure. Mirrors the server's field allowlist exactly: a field not
|
|
23
|
+
* listed here is ignored rather than stored, so the type is the whole surface.
|
|
24
|
+
* `null` clears a field and falls back to the deployment default.
|
|
25
|
+
*/
|
|
26
|
+
interface ConfigUpdate {
|
|
27
|
+
llm_provider?: LlmProvider | null;
|
|
28
|
+
/** The credential for whichever `llm_provider` is set — the field name is shared. */
|
|
29
|
+
openai_api_key?: string | null;
|
|
30
|
+
/** Only honoured alongside this key's own `openai_api_key`. */
|
|
31
|
+
openai_base_url?: string | null;
|
|
32
|
+
/** Overrides the provider's default model. */
|
|
33
|
+
chat_model?: string | null;
|
|
34
|
+
browser_provider?: Provider | null;
|
|
35
|
+
anchor_api_key?: string | null;
|
|
36
|
+
browserbase_api_key?: string | null;
|
|
37
|
+
browserbase_project_id?: string | null;
|
|
38
|
+
steel_api_key?: string | null;
|
|
39
|
+
browseruse_api_key?: string | null;
|
|
40
|
+
cdp_ws_url?: string | null;
|
|
41
|
+
captcha_solver?: CaptchaSolver | null;
|
|
42
|
+
captcha_api_key?: string | null;
|
|
43
|
+
onboarded?: string | null;
|
|
44
|
+
}
|
|
45
|
+
/** What `config.get()` returns. Secrets read back masked, never in full. */
|
|
46
|
+
interface Config extends Omit<ConfigUpdate, 'llm_provider' | 'browser_provider' | 'captcha_solver'> {
|
|
47
|
+
llm_provider?: LlmProvider | '';
|
|
48
|
+
browser_provider?: Provider | '';
|
|
49
|
+
captcha_solver?: CaptchaSolver;
|
|
50
|
+
/** What this key would actually use right now, deployment defaults included. */
|
|
51
|
+
effective: {
|
|
52
|
+
baseUrl: string;
|
|
53
|
+
model: string;
|
|
54
|
+
hasLlmKey: boolean;
|
|
55
|
+
};
|
|
56
|
+
/** True when the LLM key in play belongs to the deployment, not this key. */
|
|
57
|
+
inherited: boolean;
|
|
58
|
+
has_openai_key: boolean;
|
|
59
|
+
providers: Array<{
|
|
60
|
+
id: Provider;
|
|
61
|
+
label: string;
|
|
62
|
+
needs: string[];
|
|
63
|
+
configured: boolean;
|
|
64
|
+
}>;
|
|
65
|
+
}
|
|
12
66
|
/** Where a browser comes from. Configuration, not something a caller must know. */
|
|
13
67
|
type Provider = 'oya-cloud' | 'oya-selfhosted' | 'browseruse' | 'browserbase' | 'steel' | 'anchor' | 'cdp';
|
|
14
68
|
interface StartOptions {
|
|
@@ -58,6 +112,8 @@ interface Playbook {
|
|
|
58
112
|
name: string;
|
|
59
113
|
/** Inputs `play()` accepts; any left out reuse the recorded value. */
|
|
60
114
|
variables: string[];
|
|
115
|
+
/** What each variable was recorded with. A secret has none — it never left the page. */
|
|
116
|
+
defaults: Record<string, string>;
|
|
61
117
|
steps: number;
|
|
62
118
|
/** The same flow as a Playwright module: `export default async function run(page, vars)`. */
|
|
63
119
|
code: string;
|
|
@@ -83,12 +139,28 @@ interface PlaybookSummary extends Playbook {
|
|
|
83
139
|
healedFrom: number;
|
|
84
140
|
}) | null;
|
|
85
141
|
}
|
|
142
|
+
/**
|
|
143
|
+
* A file attached to a task value. Build it with `file()`, never by hand. Only `data`
|
|
144
|
+
* takes one: a file is not typed through a placeholder, so `secrets` has nothing to hide
|
|
145
|
+
* and rejects it.
|
|
146
|
+
*/
|
|
147
|
+
interface FileValue {
|
|
148
|
+
/** The filename the site sees. */
|
|
149
|
+
file: string;
|
|
150
|
+
/** MIME type, guessed from the extension unless you pass one. */
|
|
151
|
+
type: string;
|
|
152
|
+
/** The bytes, base64. 10MB ceiling. */
|
|
153
|
+
b64: string;
|
|
154
|
+
}
|
|
86
155
|
/**
|
|
87
156
|
* Task values, referred to as `{{name}}` in prompts. As `data` the agent can read them
|
|
88
157
|
* (to split a name or pick the right option); as `secrets` it never sees them. Either
|
|
89
158
|
* way they are typed through placeholders, so playbooks store no values.
|
|
159
|
+
*
|
|
160
|
+
* A {@link FileValue} from `file()` is the exception: the agent attaches it with its
|
|
161
|
+
* upload tool rather than typing it.
|
|
90
162
|
*/
|
|
91
|
-
type RunData = Record<string, string | number>;
|
|
163
|
+
type RunData = Record<string, string | number | FileValue>;
|
|
92
164
|
interface AttentionRequest {
|
|
93
165
|
id: string;
|
|
94
166
|
/** captcha / mfa: finish it in the live view. agent: the agent's question. heal_failed: replay and the agent both gave up. */
|
|
@@ -444,9 +516,10 @@ declare class Browser {
|
|
|
444
516
|
secrets?: RunData;
|
|
445
517
|
}): Promise<string>;
|
|
446
518
|
/**
|
|
447
|
-
* Save the last `ask()` on this browser as a named playbook.
|
|
448
|
-
*
|
|
449
|
-
*
|
|
519
|
+
* Save the last `ask()` on this browser as a named playbook. Every value that was
|
|
520
|
+
* typed, picked or clicked becomes a variable, with what the run used kept in
|
|
521
|
+
* `defaults`, so `play()` with nothing repeats the run and any one value can be
|
|
522
|
+
* swapped. `code` is the same flow as a Playwright module, to read or run yourself.
|
|
450
523
|
*/
|
|
451
524
|
toPlaybook(name: string): Promise<Playbook>;
|
|
452
525
|
/**
|
|
@@ -535,6 +608,28 @@ declare class Run {
|
|
|
535
608
|
private watch;
|
|
536
609
|
}
|
|
537
610
|
|
|
611
|
+
/**
|
|
612
|
+
* Files as task values.
|
|
613
|
+
*
|
|
614
|
+
* await browser.ask('Attach my resume', { data: { resume: await file('./cv.pdf') } });
|
|
615
|
+
*
|
|
616
|
+
* The bytes ride inline in the run request, so the agent's `upload_file` tool can put
|
|
617
|
+
* them into a page's file input. `secrets` cannot hold one — a file is never typed
|
|
618
|
+
* through a placeholder, so there is nothing to hide.
|
|
619
|
+
*/
|
|
620
|
+
|
|
621
|
+
/** Bigger than this and the server would refuse the body anyway. */
|
|
622
|
+
declare const MAX_FILE_BYTES: number;
|
|
623
|
+
/**
|
|
624
|
+
* A file for `data`. A string is a path on disk (Node only); a Blob, a File or raw bytes
|
|
625
|
+
* work anywhere. `name` is what the site sees, and `type` overrides the MIME guessed
|
|
626
|
+
* from the extension.
|
|
627
|
+
*/
|
|
628
|
+
declare function file(source: string | Uint8Array | Blob, options?: {
|
|
629
|
+
name?: string;
|
|
630
|
+
type?: string;
|
|
631
|
+
}): Promise<FileValue>;
|
|
632
|
+
|
|
538
633
|
/**
|
|
539
634
|
* @oya-ai/browser — thousands of browsers, one API.
|
|
540
635
|
*
|
|
@@ -699,14 +794,15 @@ declare class Oya {
|
|
|
699
794
|
*
|
|
700
795
|
* Bring your own LLM key (it pays for its own tokens, so no hourly chat quota applies):
|
|
701
796
|
* await oya.config.set({ llm_provider: 'gemini', openai_api_key: process.env.GEMINI_API_KEY });
|
|
702
|
-
* `llm_provider` is
|
|
703
|
-
* its default model. 'vertex' is Gemini
|
|
704
|
-
* needs no GCP project; for a
|
|
705
|
-
* `.../endpoints/openapi` and pass an
|
|
797
|
+
* `llm_provider` is the {@link LlmProvider} union, so an editor offers the choices and a
|
|
798
|
+
* typo is a compile error; `chat_model` overrides its default model. 'vertex' is Gemini
|
|
799
|
+
* Enterprise (ex-Vertex AI) in express mode, which needs no GCP project; for a
|
|
800
|
+
* project-scoped endpoint, set `openai_base_url` to `.../endpoints/openapi` and pass an
|
|
801
|
+
* OAuth access token as `openai_api_key`. See {@link ConfigUpdate} for every field.
|
|
706
802
|
*/
|
|
707
803
|
readonly config: {
|
|
708
|
-
get: <T =
|
|
709
|
-
set: <T =
|
|
804
|
+
get: <T = Config>() => Promise<T>;
|
|
805
|
+
set: <T = Config>(values: ConfigUpdate) => Promise<T>;
|
|
710
806
|
};
|
|
711
807
|
/** Saved profiles. `personas` is retained as an alias for existing clients. */
|
|
712
808
|
readonly profiles: {
|
|
@@ -764,4 +860,4 @@ declare class Oya {
|
|
|
764
860
|
private waitUntilConnected;
|
|
765
861
|
}
|
|
766
862
|
|
|
767
|
-
export { type Activity, type Analysis, type AttentionRequest, Browser, type BrowserDetail, type BrowserInfo, type CaptchaResult, type ControlCredential, type ControlEvent, type ControlOverview, type ControlRole, type ControlSession, type Element, type Fingerprint, type Health, type HumanInputAction, type MfaConfig, type MfaResult, Oya, OyaError, type OyaOptions, type PersonaInfo, type PersonaPrefs, type PlayResult, type Playbook, type PlaybookSummary, type ProjectSettings, type Provider, type ProxyCreate, type ProxyInfo, Run, type RunData, type RunInfo, type RunResult, type StartOptions, type StartResult, type StopResult, type SubmitOptions, Oya as default };
|
|
863
|
+
export { type Activity, type Analysis, type AttentionRequest, Browser, type BrowserDetail, type BrowserInfo, type CaptchaResult, type CaptchaSolver, type Config, type ConfigUpdate, type ControlCredential, type ControlEvent, type ControlOverview, type ControlRole, type ControlSession, type Element, type FileValue, type Fingerprint, type Health, type HumanInputAction, type LlmProvider, MAX_FILE_BYTES, type MfaConfig, type MfaResult, Oya, OyaError, type OyaOptions, type PersonaInfo, type PersonaPrefs, type PlayResult, type Playbook, type PlaybookSummary, type ProjectSettings, type Provider, type ProxyCreate, type ProxyInfo, Run, type RunData, type RunInfo, type RunResult, type StartOptions, type StartResult, type StopResult, type SubmitOptions, Oya as default, file };
|
package/dist/index.d.ts
CHANGED
|
@@ -9,6 +9,60 @@ declare class Http {
|
|
|
9
9
|
}
|
|
10
10
|
|
|
11
11
|
/** Everything the API returns or accepts, in one place. */
|
|
12
|
+
/** Which model drives `ask()` and the chat API. */
|
|
13
|
+
type LlmProvider = 'openai' | 'anthropic'
|
|
14
|
+
/** Gemini via AI Studio. */
|
|
15
|
+
| 'gemini'
|
|
16
|
+
/** Gemini Enterprise, ex-Vertex AI. Express mode by default; set `openai_base_url` to a
|
|
17
|
+
* project-scoped `.../endpoints/openapi` endpoint to use an enterprise project. */
|
|
18
|
+
| 'vertex';
|
|
19
|
+
/** Empty disables solving. */
|
|
20
|
+
type CaptchaSolver = 'capsolver' | '2captcha' | '';
|
|
21
|
+
/**
|
|
22
|
+
* What a key can configure. Mirrors the server's field allowlist exactly: a field not
|
|
23
|
+
* listed here is ignored rather than stored, so the type is the whole surface.
|
|
24
|
+
* `null` clears a field and falls back to the deployment default.
|
|
25
|
+
*/
|
|
26
|
+
interface ConfigUpdate {
|
|
27
|
+
llm_provider?: LlmProvider | null;
|
|
28
|
+
/** The credential for whichever `llm_provider` is set — the field name is shared. */
|
|
29
|
+
openai_api_key?: string | null;
|
|
30
|
+
/** Only honoured alongside this key's own `openai_api_key`. */
|
|
31
|
+
openai_base_url?: string | null;
|
|
32
|
+
/** Overrides the provider's default model. */
|
|
33
|
+
chat_model?: string | null;
|
|
34
|
+
browser_provider?: Provider | null;
|
|
35
|
+
anchor_api_key?: string | null;
|
|
36
|
+
browserbase_api_key?: string | null;
|
|
37
|
+
browserbase_project_id?: string | null;
|
|
38
|
+
steel_api_key?: string | null;
|
|
39
|
+
browseruse_api_key?: string | null;
|
|
40
|
+
cdp_ws_url?: string | null;
|
|
41
|
+
captcha_solver?: CaptchaSolver | null;
|
|
42
|
+
captcha_api_key?: string | null;
|
|
43
|
+
onboarded?: string | null;
|
|
44
|
+
}
|
|
45
|
+
/** What `config.get()` returns. Secrets read back masked, never in full. */
|
|
46
|
+
interface Config extends Omit<ConfigUpdate, 'llm_provider' | 'browser_provider' | 'captcha_solver'> {
|
|
47
|
+
llm_provider?: LlmProvider | '';
|
|
48
|
+
browser_provider?: Provider | '';
|
|
49
|
+
captcha_solver?: CaptchaSolver;
|
|
50
|
+
/** What this key would actually use right now, deployment defaults included. */
|
|
51
|
+
effective: {
|
|
52
|
+
baseUrl: string;
|
|
53
|
+
model: string;
|
|
54
|
+
hasLlmKey: boolean;
|
|
55
|
+
};
|
|
56
|
+
/** True when the LLM key in play belongs to the deployment, not this key. */
|
|
57
|
+
inherited: boolean;
|
|
58
|
+
has_openai_key: boolean;
|
|
59
|
+
providers: Array<{
|
|
60
|
+
id: Provider;
|
|
61
|
+
label: string;
|
|
62
|
+
needs: string[];
|
|
63
|
+
configured: boolean;
|
|
64
|
+
}>;
|
|
65
|
+
}
|
|
12
66
|
/** Where a browser comes from. Configuration, not something a caller must know. */
|
|
13
67
|
type Provider = 'oya-cloud' | 'oya-selfhosted' | 'browseruse' | 'browserbase' | 'steel' | 'anchor' | 'cdp';
|
|
14
68
|
interface StartOptions {
|
|
@@ -58,6 +112,8 @@ interface Playbook {
|
|
|
58
112
|
name: string;
|
|
59
113
|
/** Inputs `play()` accepts; any left out reuse the recorded value. */
|
|
60
114
|
variables: string[];
|
|
115
|
+
/** What each variable was recorded with. A secret has none — it never left the page. */
|
|
116
|
+
defaults: Record<string, string>;
|
|
61
117
|
steps: number;
|
|
62
118
|
/** The same flow as a Playwright module: `export default async function run(page, vars)`. */
|
|
63
119
|
code: string;
|
|
@@ -83,12 +139,28 @@ interface PlaybookSummary extends Playbook {
|
|
|
83
139
|
healedFrom: number;
|
|
84
140
|
}) | null;
|
|
85
141
|
}
|
|
142
|
+
/**
|
|
143
|
+
* A file attached to a task value. Build it with `file()`, never by hand. Only `data`
|
|
144
|
+
* takes one: a file is not typed through a placeholder, so `secrets` has nothing to hide
|
|
145
|
+
* and rejects it.
|
|
146
|
+
*/
|
|
147
|
+
interface FileValue {
|
|
148
|
+
/** The filename the site sees. */
|
|
149
|
+
file: string;
|
|
150
|
+
/** MIME type, guessed from the extension unless you pass one. */
|
|
151
|
+
type: string;
|
|
152
|
+
/** The bytes, base64. 10MB ceiling. */
|
|
153
|
+
b64: string;
|
|
154
|
+
}
|
|
86
155
|
/**
|
|
87
156
|
* Task values, referred to as `{{name}}` in prompts. As `data` the agent can read them
|
|
88
157
|
* (to split a name or pick the right option); as `secrets` it never sees them. Either
|
|
89
158
|
* way they are typed through placeholders, so playbooks store no values.
|
|
159
|
+
*
|
|
160
|
+
* A {@link FileValue} from `file()` is the exception: the agent attaches it with its
|
|
161
|
+
* upload tool rather than typing it.
|
|
90
162
|
*/
|
|
91
|
-
type RunData = Record<string, string | number>;
|
|
163
|
+
type RunData = Record<string, string | number | FileValue>;
|
|
92
164
|
interface AttentionRequest {
|
|
93
165
|
id: string;
|
|
94
166
|
/** captcha / mfa: finish it in the live view. agent: the agent's question. heal_failed: replay and the agent both gave up. */
|
|
@@ -444,9 +516,10 @@ declare class Browser {
|
|
|
444
516
|
secrets?: RunData;
|
|
445
517
|
}): Promise<string>;
|
|
446
518
|
/**
|
|
447
|
-
* Save the last `ask()` on this browser as a named playbook.
|
|
448
|
-
*
|
|
449
|
-
*
|
|
519
|
+
* Save the last `ask()` on this browser as a named playbook. Every value that was
|
|
520
|
+
* typed, picked or clicked becomes a variable, with what the run used kept in
|
|
521
|
+
* `defaults`, so `play()` with nothing repeats the run and any one value can be
|
|
522
|
+
* swapped. `code` is the same flow as a Playwright module, to read or run yourself.
|
|
450
523
|
*/
|
|
451
524
|
toPlaybook(name: string): Promise<Playbook>;
|
|
452
525
|
/**
|
|
@@ -535,6 +608,28 @@ declare class Run {
|
|
|
535
608
|
private watch;
|
|
536
609
|
}
|
|
537
610
|
|
|
611
|
+
/**
|
|
612
|
+
* Files as task values.
|
|
613
|
+
*
|
|
614
|
+
* await browser.ask('Attach my resume', { data: { resume: await file('./cv.pdf') } });
|
|
615
|
+
*
|
|
616
|
+
* The bytes ride inline in the run request, so the agent's `upload_file` tool can put
|
|
617
|
+
* them into a page's file input. `secrets` cannot hold one — a file is never typed
|
|
618
|
+
* through a placeholder, so there is nothing to hide.
|
|
619
|
+
*/
|
|
620
|
+
|
|
621
|
+
/** Bigger than this and the server would refuse the body anyway. */
|
|
622
|
+
declare const MAX_FILE_BYTES: number;
|
|
623
|
+
/**
|
|
624
|
+
* A file for `data`. A string is a path on disk (Node only); a Blob, a File or raw bytes
|
|
625
|
+
* work anywhere. `name` is what the site sees, and `type` overrides the MIME guessed
|
|
626
|
+
* from the extension.
|
|
627
|
+
*/
|
|
628
|
+
declare function file(source: string | Uint8Array | Blob, options?: {
|
|
629
|
+
name?: string;
|
|
630
|
+
type?: string;
|
|
631
|
+
}): Promise<FileValue>;
|
|
632
|
+
|
|
538
633
|
/**
|
|
539
634
|
* @oya-ai/browser — thousands of browsers, one API.
|
|
540
635
|
*
|
|
@@ -699,14 +794,15 @@ declare class Oya {
|
|
|
699
794
|
*
|
|
700
795
|
* Bring your own LLM key (it pays for its own tokens, so no hourly chat quota applies):
|
|
701
796
|
* await oya.config.set({ llm_provider: 'gemini', openai_api_key: process.env.GEMINI_API_KEY });
|
|
702
|
-
* `llm_provider` is
|
|
703
|
-
* its default model. 'vertex' is Gemini
|
|
704
|
-
* needs no GCP project; for a
|
|
705
|
-
* `.../endpoints/openapi` and pass an
|
|
797
|
+
* `llm_provider` is the {@link LlmProvider} union, so an editor offers the choices and a
|
|
798
|
+
* typo is a compile error; `chat_model` overrides its default model. 'vertex' is Gemini
|
|
799
|
+
* Enterprise (ex-Vertex AI) in express mode, which needs no GCP project; for a
|
|
800
|
+
* project-scoped endpoint, set `openai_base_url` to `.../endpoints/openapi` and pass an
|
|
801
|
+
* OAuth access token as `openai_api_key`. See {@link ConfigUpdate} for every field.
|
|
706
802
|
*/
|
|
707
803
|
readonly config: {
|
|
708
|
-
get: <T =
|
|
709
|
-
set: <T =
|
|
804
|
+
get: <T = Config>() => Promise<T>;
|
|
805
|
+
set: <T = Config>(values: ConfigUpdate) => Promise<T>;
|
|
710
806
|
};
|
|
711
807
|
/** Saved profiles. `personas` is retained as an alias for existing clients. */
|
|
712
808
|
readonly profiles: {
|
|
@@ -764,4 +860,4 @@ declare class Oya {
|
|
|
764
860
|
private waitUntilConnected;
|
|
765
861
|
}
|
|
766
862
|
|
|
767
|
-
export { type Activity, type Analysis, type AttentionRequest, Browser, type BrowserDetail, type BrowserInfo, type CaptchaResult, type ControlCredential, type ControlEvent, type ControlOverview, type ControlRole, type ControlSession, type Element, type Fingerprint, type Health, type HumanInputAction, type MfaConfig, type MfaResult, Oya, OyaError, type OyaOptions, type PersonaInfo, type PersonaPrefs, type PlayResult, type Playbook, type PlaybookSummary, type ProjectSettings, type Provider, type ProxyCreate, type ProxyInfo, Run, type RunData, type RunInfo, type RunResult, type StartOptions, type StartResult, type StopResult, type SubmitOptions, Oya as default };
|
|
863
|
+
export { type Activity, type Analysis, type AttentionRequest, Browser, type BrowserDetail, type BrowserInfo, type CaptchaResult, type CaptchaSolver, type Config, type ConfigUpdate, type ControlCredential, type ControlEvent, type ControlOverview, type ControlRole, type ControlSession, type Element, type FileValue, type Fingerprint, type Health, type HumanInputAction, type LlmProvider, MAX_FILE_BYTES, type MfaConfig, type MfaResult, Oya, OyaError, type OyaOptions, type PersonaInfo, type PersonaPrefs, type PlayResult, type Playbook, type PlaybookSummary, type ProjectSettings, type Provider, type ProxyCreate, type ProxyInfo, Run, type RunData, type RunInfo, type RunResult, type StartOptions, type StartResult, type StopResult, type SubmitOptions, Oya as default, file };
|
package/dist/index.js
CHANGED
|
@@ -173,9 +173,10 @@ var Browser = class {
|
|
|
173
173
|
return res.text;
|
|
174
174
|
}
|
|
175
175
|
/**
|
|
176
|
-
* Save the last `ask()` on this browser as a named playbook.
|
|
177
|
-
*
|
|
178
|
-
*
|
|
176
|
+
* Save the last `ask()` on this browser as a named playbook. Every value that was
|
|
177
|
+
* typed, picked or clicked becomes a variable, with what the run used kept in
|
|
178
|
+
* `defaults`, so `play()` with nothing repeats the run and any one value can be
|
|
179
|
+
* swapped. `code` is the same flow as a Playwright module, to read or run yourself.
|
|
179
180
|
*/
|
|
180
181
|
toPlaybook(name) {
|
|
181
182
|
return this.http.request("POST", `/api/browsers/${this.id}/playbooks`, { name }, 12e4);
|
|
@@ -346,6 +347,63 @@ var Run = class {
|
|
|
346
347
|
}
|
|
347
348
|
};
|
|
348
349
|
|
|
350
|
+
// src/file.ts
|
|
351
|
+
var MAX_FILE_BYTES = 10 * 1024 * 1024;
|
|
352
|
+
var NODE_FS = "node:fs/promises";
|
|
353
|
+
var MIME = {
|
|
354
|
+
pdf: "application/pdf",
|
|
355
|
+
png: "image/png",
|
|
356
|
+
jpg: "image/jpeg",
|
|
357
|
+
jpeg: "image/jpeg",
|
|
358
|
+
gif: "image/gif",
|
|
359
|
+
webp: "image/webp",
|
|
360
|
+
svg: "image/svg+xml",
|
|
361
|
+
heic: "image/heic",
|
|
362
|
+
txt: "text/plain",
|
|
363
|
+
csv: "text/csv",
|
|
364
|
+
json: "application/json",
|
|
365
|
+
xml: "application/xml",
|
|
366
|
+
html: "text/html",
|
|
367
|
+
doc: "application/msword",
|
|
368
|
+
docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
369
|
+
xls: "application/vnd.ms-excel",
|
|
370
|
+
xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
371
|
+
ppt: "application/vnd.ms-powerpoint",
|
|
372
|
+
pptx: "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
|
373
|
+
zip: "application/zip"
|
|
374
|
+
};
|
|
375
|
+
var base64 = (bytes) => {
|
|
376
|
+
const buffer = globalThis.Buffer;
|
|
377
|
+
if (buffer) return buffer.from(bytes).toString("base64");
|
|
378
|
+
let binary = "";
|
|
379
|
+
for (let i = 0; i < bytes.length; i += 8192) binary += String.fromCharCode(...bytes.subarray(i, i + 8192));
|
|
380
|
+
return btoa(binary);
|
|
381
|
+
};
|
|
382
|
+
async function file(source, options = {}) {
|
|
383
|
+
let bytes;
|
|
384
|
+
let name = options.name;
|
|
385
|
+
if (typeof source === "string") {
|
|
386
|
+
const fs = await import(
|
|
387
|
+
/* webpackIgnore: true */
|
|
388
|
+
/* @vite-ignore */
|
|
389
|
+
NODE_FS
|
|
390
|
+
);
|
|
391
|
+
bytes = new Uint8Array(await fs.readFile(source));
|
|
392
|
+
name ||= source.split(/[\\/]/).pop() || "file";
|
|
393
|
+
} else if (source instanceof Uint8Array) {
|
|
394
|
+
bytes = source;
|
|
395
|
+
} else {
|
|
396
|
+
bytes = new Uint8Array(await source.arrayBuffer());
|
|
397
|
+
name ||= source.name;
|
|
398
|
+
}
|
|
399
|
+
name ||= "file";
|
|
400
|
+
if (bytes.length > MAX_FILE_BYTES) {
|
|
401
|
+
throw new Error(`${name} is ${Math.round(bytes.length / 1024 / 1024)}MB; the limit for a task file is ${MAX_FILE_BYTES / 1024 / 1024}MB.`);
|
|
402
|
+
}
|
|
403
|
+
const ext = name.includes(".") ? name.split(".").pop().toLowerCase() : "";
|
|
404
|
+
return { file: name, type: options.type || MIME[ext] || "application/octet-stream", b64: base64(bytes) };
|
|
405
|
+
}
|
|
406
|
+
|
|
349
407
|
// src/index.ts
|
|
350
408
|
var DEFAULT_BASE_URL = "https://browser.getoya.ai";
|
|
351
409
|
var READY_POLL_MS = 2e3;
|
|
@@ -476,10 +534,11 @@ var Oya = class {
|
|
|
476
534
|
*
|
|
477
535
|
* Bring your own LLM key (it pays for its own tokens, so no hourly chat quota applies):
|
|
478
536
|
* await oya.config.set({ llm_provider: 'gemini', openai_api_key: process.env.GEMINI_API_KEY });
|
|
479
|
-
* `llm_provider` is
|
|
480
|
-
* its default model. 'vertex' is Gemini
|
|
481
|
-
* needs no GCP project; for a
|
|
482
|
-
* `.../endpoints/openapi` and pass an
|
|
537
|
+
* `llm_provider` is the {@link LlmProvider} union, so an editor offers the choices and a
|
|
538
|
+
* typo is a compile error; `chat_model` overrides its default model. 'vertex' is Gemini
|
|
539
|
+
* Enterprise (ex-Vertex AI) in express mode, which needs no GCP project; for a
|
|
540
|
+
* project-scoped endpoint, set `openai_base_url` to `.../endpoints/openapi` and pass an
|
|
541
|
+
* OAuth access token as `openai_api_key`. See {@link ConfigUpdate} for every field.
|
|
483
542
|
*/
|
|
484
543
|
config = {
|
|
485
544
|
get: () => this.http.request("GET", "/api/config"),
|
|
@@ -509,8 +568,10 @@ var Oya = class {
|
|
|
509
568
|
var index_default = Oya;
|
|
510
569
|
export {
|
|
511
570
|
Browser,
|
|
571
|
+
MAX_FILE_BYTES,
|
|
512
572
|
Oya,
|
|
513
573
|
OyaError,
|
|
514
574
|
Run,
|
|
515
|
-
index_default as default
|
|
575
|
+
index_default as default,
|
|
576
|
+
file
|
|
516
577
|
};
|
package/package.json
CHANGED