@aluvia-connect/agent-connect 1.0.4 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -72
- package/dist/cjs/src/index.js +51 -115
- package/dist/esm/src/index.js +50 -114
- package/dist/types/src/index.d.ts +19 -32
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Agent Connect
|
|
2
2
|
|
|
3
|
-
[](https://badge.fury.io/js/@aluvia-connect%2Fagent-connect)
|
|
4
4
|
[](https://opensource.org/licenses/MIT)
|
|
5
5
|
[](https://www.typescriptlang.org)
|
|
6
6
|
|
|
@@ -24,67 +24,83 @@ pnpm add @aluvia-connect/agent-connect
|
|
|
24
24
|
|
|
25
25
|
## Quick Start
|
|
26
26
|
|
|
27
|
-
```
|
|
27
|
+
```ts
|
|
28
28
|
import { chromium } from "playwright";
|
|
29
|
-
import {
|
|
29
|
+
import { agentConnect, startDynamicProxy } from "@aluvia-connect/agent-connect";
|
|
30
30
|
|
|
31
|
-
|
|
31
|
+
// Start local proxy-chain server (random free port)
|
|
32
|
+
const dyn = await startDynamicProxy();
|
|
33
|
+
|
|
34
|
+
// Launch browser using ONLY the local proxy initially (direct connection upstream)
|
|
35
|
+
const browser = await chromium.launch({ proxy: { server: dyn.url } });
|
|
32
36
|
const context = await browser.newContext();
|
|
33
37
|
const page = await context.newPage();
|
|
34
38
|
|
|
35
|
-
const {
|
|
36
|
-
|
|
37
|
-
|
|
39
|
+
const { page: samePage } = await agentConnect(page, {
|
|
40
|
+
dynamicProxy: dyn,
|
|
41
|
+
maxRetries: 2,
|
|
42
|
+
retryOn: ["Timeout", /net::ERR/],
|
|
43
|
+
onProxyLoaded: (p) => console.log("Upstream proxy loaded", p.server),
|
|
44
|
+
onRetry: (a, m) => console.log(`Retry ${a}/${m}`),
|
|
45
|
+
}).goto("https://blocked-website.example");
|
|
38
46
|
|
|
39
|
-
console.log(
|
|
47
|
+
console.log(await samePage.title());
|
|
40
48
|
await browser.close();
|
|
41
49
|
```
|
|
42
50
|
|
|
43
|
-
|
|
51
|
+
Notes:
|
|
44
52
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
53
|
+
- The first attempt is direct (no upstream proxy). On failure, we fetch a proxy and call `dynamicProxy.setUpstream()` internally.
|
|
54
|
+
- Subsequent retries reuse the same browser & page; cookies and session data persist.
|
|
55
|
+
- Provide your own `proxyProvider` if you do not want to use the Aluvia API.
|
|
56
|
+
- The dynamic proxy closes automatically when `browser.close()` is called. You can also call `dyn.close()` manually.
|
|
48
57
|
|
|
49
|
-
|
|
50
|
-
|
|
58
|
+
You can integrate this with any proxy API or local pool, as long as it returns a `server`, `username`, and `password`.
|
|
59
|
+
|
|
60
|
+
## Aluvia Token Setup
|
|
61
|
+
|
|
62
|
+
This SDK uses an Aluvia token to fetch proxies when retries occur. You can find your token on your Aluvia account's [credentials](https://dashboard.aluvia.io/credentials) page.
|
|
63
|
+
|
|
64
|
+
Set your token key in a `.env` file:
|
|
65
|
+
|
|
66
|
+
```env
|
|
67
|
+
ALUVIA_TOKEN=your_aluvia_token
|
|
51
68
|
```
|
|
52
69
|
|
|
53
70
|
## Configuration
|
|
54
71
|
|
|
55
|
-
You can control how `
|
|
72
|
+
You can control how `agentConnect` behaves using environment variables or options passed in code.
|
|
56
73
|
The environment variables set defaults globally, while the TypeScript options let you override them per call.
|
|
57
74
|
|
|
58
75
|
### Environment Variables
|
|
59
76
|
|
|
60
77
|
| Variable | Description | Default |
|
|
61
|
-
|
|
62
|
-
| `
|
|
63
|
-
| `ALUVIA_MAX_RETRIES` | Number of retry attempts after the first failed navigation. | `
|
|
78
|
+
|----------------------| ---------------------------------------------------------------------------------------- |-----------------------------------------|
|
|
79
|
+
| `ALUVIA_TOKEN` | Required unless you provide a custom `proxyProvider`. Used to fetch proxies from Aluvia. | _none_ |
|
|
80
|
+
| `ALUVIA_MAX_RETRIES` | Number of retry attempts after the first failed navigation. | `2` |
|
|
64
81
|
| `ALUVIA_BACKOFF_MS` | Base delay (ms) between retries, grows exponentially with jitter. | `300` |
|
|
65
82
|
| `ALUVIA_RETRY_ON` | Comma-separated list of retryable error substrings. | `ECONNRESET,ETIMEDOUT,net::ERR,Timeout` |
|
|
66
83
|
|
|
67
84
|
#### Example `.env`
|
|
68
85
|
|
|
69
86
|
```env
|
|
70
|
-
|
|
71
|
-
ALUVIA_MAX_RETRIES=
|
|
87
|
+
ALUVIA_TOKEN=your_aluvia_token
|
|
88
|
+
ALUVIA_MAX_RETRIES=1
|
|
72
89
|
ALUVIA_BACKOFF_MS=500
|
|
73
90
|
ALUVIA_RETRY_ON=ECONNRESET,ETIMEDOUT,net::ERR,Timeout
|
|
74
91
|
```
|
|
75
92
|
|
|
76
|
-
###
|
|
93
|
+
### Options
|
|
77
94
|
|
|
78
|
-
You can also configure behavior programmatically by passing options to `
|
|
95
|
+
You can also configure behavior programmatically by passing options to `agentConnect()`.
|
|
79
96
|
|
|
80
97
|
```typescript
|
|
81
|
-
import {
|
|
98
|
+
import { agentConnect } from "@aluvia-connect/agent-connect";
|
|
82
99
|
|
|
83
|
-
const { response, page } = await
|
|
100
|
+
const { response, page } = await agentConnect(page, {
|
|
84
101
|
maxRetries: 3,
|
|
85
102
|
backoffMs: 500,
|
|
86
103
|
retryOn: ["ECONNRESET", /403/],
|
|
87
|
-
closeOldBrowser: false,
|
|
88
104
|
onRetry: (attempt, maxRetries, lastError) => {
|
|
89
105
|
console.log(
|
|
90
106
|
`Retry attempt ${attempt} of ${maxRetries} due to error:`,
|
|
@@ -100,11 +116,10 @@ const { response, page } = await retryWithProxy(page, {
|
|
|
100
116
|
#### Available Options
|
|
101
117
|
|
|
102
118
|
| Option | Type | Default | Description |
|
|
103
|
-
| ----------------- | ------------------------------------------------------------------------------------
|
|
104
|
-
| `maxRetries` | `number` | `process.env.ALUVIA_MAX_RETRIES` or `
|
|
119
|
+
| ----------------- | ------------------------------------------------------------------------------------ |------------------------------------------| ------------------------------------------------------------------------------------------------------------- |
|
|
120
|
+
| `maxRetries` | `number` | `process.env.ALUVIA_MAX_RETRIES` or `2` | Number of retry attempts after the first failure. |
|
|
105
121
|
| `backoffMs` | `number` | `process.env.ALUVIA_BACKOFF_MS` or `300` | Base delay (in ms) between retries, grows exponentially with jitter. |
|
|
106
122
|
| `retryOn` | `(string \| RegExp)[]` | `process.env.ALUVIA_RETRY_ON` | Error patterns considered retryable. |
|
|
107
|
-
| `closeOldBrowser` | `boolean` | `true` | Whether to close the old browser when relaunching. |
|
|
108
123
|
| `proxyProvider` | `ProxyProvider` | Uses Aluvia SDK | Custom proxy provider that returns proxy credentials. |
|
|
109
124
|
| `onRetry` | `(attempt: number, maxRetries: number, lastError: unknown) => void \| Promise<void>` | `undefined` | Callback invoked before each retry attempt. |
|
|
110
125
|
| `onProxyLoaded` | `(proxy: ProxySettings) => void \| Promise<void>` | `undefined` | Callback fired after a proxy has been successfully fetched (either from the Aluvia API or a custom provider). |
|
|
@@ -122,59 +137,17 @@ const myProxyProvider = {
|
|
|
122
137
|
},
|
|
123
138
|
};
|
|
124
139
|
|
|
125
|
-
const { response, page } = await
|
|
140
|
+
const { response, page } = await agentConnect(page, {
|
|
126
141
|
proxyProvider: myProxyProvider,
|
|
127
142
|
maxRetries: 3,
|
|
128
143
|
});
|
|
129
144
|
```
|
|
130
145
|
|
|
131
|
-
## Dynamic Proxy (No Browser Relaunch)
|
|
132
|
-
|
|
133
|
-
To avoid relaunching the browser for each retry you can run a local proxy that dynamically swaps its upstream. This keeps all pages, contexts and session state intact.
|
|
134
|
-
|
|
135
|
-
1. Start the dynamic proxy.
|
|
136
|
-
2. Launch Playwright pointing at its local address.
|
|
137
|
-
3. Pass the dynamic proxy into `retryWithProxy`.
|
|
138
|
-
4. On a retryable failure the upstream proxy is fetched and swapped; navigation is re-attempted on the same page instance.
|
|
139
|
-
|
|
140
|
-
```ts
|
|
141
|
-
import { chromium } from "playwright";
|
|
142
|
-
import { retryWithProxy, startDynamicProxy } from "@aluvia-connect/agent-connect";
|
|
143
|
-
|
|
144
|
-
// Start local proxy-chain server (random free port)
|
|
145
|
-
const dyn = await startDynamicProxy();
|
|
146
|
-
|
|
147
|
-
// Launch browser using ONLY the local proxy initially (direct connection upstream)
|
|
148
|
-
const browser = await chromium.launch({ proxy: { server: dyn.url } });
|
|
149
|
-
const context = await browser.newContext();
|
|
150
|
-
const page = await context.newPage();
|
|
151
|
-
|
|
152
|
-
const { page: samePage } = await retryWithProxy(page, {
|
|
153
|
-
dynamicProxy: dyn,
|
|
154
|
-
maxRetries: 2,
|
|
155
|
-
retryOn: ["Timeout", /net::ERR/],
|
|
156
|
-
onProxyLoaded: (p) => console.log("Upstream proxy loaded", p.server),
|
|
157
|
-
onRetry: (a, m) => console.log(`Dynamic retry ${a}/${m}`),
|
|
158
|
-
}).goto("https://blocked-website.example");
|
|
159
|
-
|
|
160
|
-
console.log(await samePage.title());
|
|
161
|
-
await dyn.close();
|
|
162
|
-
await browser.close();
|
|
163
|
-
```
|
|
164
|
-
|
|
165
|
-
Notes:
|
|
166
|
-
|
|
167
|
-
- The first attempt is direct (no upstream proxy). On failure we fetch a proxy and call `dynamicProxy.setUpstream()` internally.
|
|
168
|
-
- Subsequent retries reuse the same browser & page; cookies and session data persist.
|
|
169
|
-
- Provide your own `proxyProvider` if you do not want to use the Aluvia API.
|
|
170
|
-
|
|
171
|
-
You can integrate this with any proxy API or local pool, as long as it returns a `server`, `username`, and `password`.
|
|
172
|
-
|
|
173
146
|
## Requirements
|
|
174
147
|
|
|
175
148
|
- Node.js >= 18
|
|
176
149
|
- Playwright
|
|
177
|
-
- Aluvia
|
|
150
|
+
- Aluvia token (_if not using a custom proxy provider_)
|
|
178
151
|
|
|
179
152
|
## About Aluvia
|
|
180
153
|
|
package/dist/cjs/src/index.js
CHANGED
|
@@ -34,11 +34,11 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
36
|
exports.AluviaError = void 0;
|
|
37
|
-
exports.
|
|
37
|
+
exports.agentConnect = agentConnect;
|
|
38
38
|
exports.startDynamicProxy = startDynamicProxy;
|
|
39
39
|
const proxy_chain_1 = require("proxy-chain");
|
|
40
40
|
const DEFAULT_GOTO_TIMEOUT_MS = 15000;
|
|
41
|
-
const ENV_MAX_RETRIES = Math.max(0, parseInt(process.env.ALUVIA_MAX_RETRIES || "
|
|
41
|
+
const ENV_MAX_RETRIES = Math.max(0, parseInt(process.env.ALUVIA_MAX_RETRIES || "2", 10)); // prettier-ignore
|
|
42
42
|
const ENV_BACKOFF_MS = Math.max(0, parseInt(process.env.ALUVIA_BACKOFF_MS || "300", 10)); // prettier-ignore
|
|
43
43
|
const ENV_RETRY_ON = (process.env.ALUVIA_RETRY_ON ?? "ECONNRESET,ETIMEDOUT,net::ERR,Timeout")
|
|
44
44
|
.split(",")
|
|
@@ -55,6 +55,7 @@ var AluviaErrorCode;
|
|
|
55
55
|
AluviaErrorCode["ProxyFetchFailed"] = "ALUVIA_PROXY_FETCH_FAILED";
|
|
56
56
|
AluviaErrorCode["InsufficientBalance"] = "ALUVIA_INSUFFICIENT_BALANCE";
|
|
57
57
|
AluviaErrorCode["BalanceFetchFailed"] = "ALUVIA_BALANCE_FETCH_FAILED";
|
|
58
|
+
AluviaErrorCode["NoDynamicProxy"] = "ALUVIA_NO_DYNAMIC_PROXY";
|
|
58
59
|
})(AluviaErrorCode || (AluviaErrorCode = {}));
|
|
59
60
|
class AluviaError extends Error {
|
|
60
61
|
constructor(message, code) {
|
|
@@ -66,9 +67,9 @@ class AluviaError extends Error {
|
|
|
66
67
|
exports.AluviaError = AluviaError;
|
|
67
68
|
let aluviaClient; // lazy-loaded Aluvia client instance
|
|
68
69
|
async function getAluviaProxy() {
|
|
69
|
-
const apiKey = process.env.
|
|
70
|
+
const apiKey = process.env.ALUVIA_TOKEN || "";
|
|
70
71
|
if (!apiKey) {
|
|
71
|
-
throw new AluviaError("Missing
|
|
72
|
+
throw new AluviaError("Missing ALUVIA_TOKEN environment variable.", AluviaErrorCode.NoApiKey);
|
|
72
73
|
}
|
|
73
74
|
if (!aluviaClient) {
|
|
74
75
|
// Dynamic import to play nicely with test mocks (avoids top-level evaluation before vi.mock)
|
|
@@ -80,16 +81,17 @@ async function getAluviaProxy() {
|
|
|
80
81
|
if (!proxy) {
|
|
81
82
|
throw new AluviaError("Failed to obtain a proxy for retry attempts. Check your balance and proxy pool at https://dashboard.aluvia.io/.", AluviaErrorCode.NoProxy);
|
|
82
83
|
}
|
|
84
|
+
const sessionId = generateSessionId();
|
|
83
85
|
return {
|
|
84
86
|
server: `http://${proxy.host}:${proxy.httpPort}`,
|
|
85
|
-
username: proxy.username
|
|
87
|
+
username: `${proxy.username}-session-${sessionId}`,
|
|
86
88
|
password: proxy.password,
|
|
87
89
|
};
|
|
88
90
|
}
|
|
89
91
|
async function getAluviaBalance() {
|
|
90
|
-
const apiKey = process.env.
|
|
92
|
+
const apiKey = process.env.ALUVIA_TOKEN || "";
|
|
91
93
|
if (!apiKey) {
|
|
92
|
-
throw new AluviaError("Missing
|
|
94
|
+
throw new AluviaError("Missing ALUVIA_TOKEN environment variable.", AluviaErrorCode.NoApiKey);
|
|
93
95
|
}
|
|
94
96
|
const response = await fetch("https://api.aluvia.io/account/status", {
|
|
95
97
|
headers: {
|
|
@@ -119,47 +121,16 @@ function compileRetryable(patterns = DEFAULT_RETRY_PATTERNS) {
|
|
|
119
121
|
: msg.includes(p) || code.includes(p) || name.includes(p));
|
|
120
122
|
};
|
|
121
123
|
}
|
|
122
|
-
function
|
|
123
|
-
|
|
124
|
-
const browserType = browser?.browserType?.();
|
|
125
|
-
if (!browserType) {
|
|
126
|
-
throw new Error("Cannot infer BrowserType from page");
|
|
127
|
-
}
|
|
128
|
-
return browserType;
|
|
129
|
-
}
|
|
130
|
-
async function inferContextDefaults(page) {
|
|
131
|
-
const context = page.context();
|
|
132
|
-
const options = context._options;
|
|
133
|
-
return options ?? {};
|
|
134
|
-
}
|
|
135
|
-
function inferLaunchDefaults(page) {
|
|
136
|
-
const browser = page.context().browser();
|
|
137
|
-
const options = browser._options;
|
|
138
|
-
return options ?? {};
|
|
139
|
-
}
|
|
140
|
-
async function relaunchWithProxy(proxy, oldPage, closeOldBrowser = true) {
|
|
141
|
-
const browserType = inferBrowserTypeFromPage(oldPage);
|
|
142
|
-
const launchDefaults = inferLaunchDefaults(oldPage);
|
|
143
|
-
const contextDefaults = await inferContextDefaults(oldPage);
|
|
144
|
-
if (closeOldBrowser) {
|
|
145
|
-
const oldBrowser = oldPage.context().browser();
|
|
146
|
-
try {
|
|
147
|
-
await oldBrowser?.close();
|
|
148
|
-
}
|
|
149
|
-
catch { }
|
|
150
|
-
}
|
|
151
|
-
const retryLaunch = {
|
|
152
|
-
...launchDefaults,
|
|
153
|
-
proxy,
|
|
154
|
-
};
|
|
155
|
-
const browser = await browserType.launch(retryLaunch);
|
|
156
|
-
const context = await browser.newContext(contextDefaults);
|
|
157
|
-
const page = await context.newPage();
|
|
158
|
-
return { page };
|
|
124
|
+
function generateSessionId() {
|
|
125
|
+
return Math.random().toString(36).substring(2, 10);
|
|
159
126
|
}
|
|
160
127
|
const GOTO_ORIGINAL = Symbol.for("aluvia.gotoOriginal");
|
|
161
|
-
|
|
162
|
-
|
|
128
|
+
const CONTEXT_LISTENER_ATTACHED = new WeakSet();
|
|
129
|
+
function agentConnect(page, options) {
|
|
130
|
+
const { dynamicProxy, maxRetries = ENV_MAX_RETRIES, backoffMs = ENV_BACKOFF_MS, retryOn = DEFAULT_RETRY_PATTERNS, proxyProvider, onRetry, onProxyLoaded, } = options ?? {};
|
|
131
|
+
if (!dynamicProxy) {
|
|
132
|
+
throw new AluviaError("No dynamic proxy supplied to agentConnect", AluviaErrorCode.NoDynamicProxy);
|
|
133
|
+
}
|
|
163
134
|
const isRetryable = compileRetryable(retryOn);
|
|
164
135
|
/** Prefer unpatched goto to avoid recursion */
|
|
165
136
|
const getRawGoto = (p) => (p[GOTO_ORIGINAL]?.bind(p) ?? p.goto.bind(p));
|
|
@@ -168,6 +139,17 @@ function retryWithProxy(page, options) {
|
|
|
168
139
|
const run = async () => {
|
|
169
140
|
let basePage = page;
|
|
170
141
|
let lastErr;
|
|
142
|
+
// One-time attach context close listener to shut down dynamic proxy
|
|
143
|
+
if (dynamicProxy) {
|
|
144
|
+
const ctx = basePage.context();
|
|
145
|
+
if (!CONTEXT_LISTENER_ATTACHED.has(ctx)) {
|
|
146
|
+
ctx.on('close', async () => { try {
|
|
147
|
+
await dynamicProxy.close();
|
|
148
|
+
}
|
|
149
|
+
catch { } });
|
|
150
|
+
CONTEXT_LISTENER_ATTACHED.add(ctx);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
171
153
|
// First attempt without proxy
|
|
172
154
|
try {
|
|
173
155
|
const response = await getRawGoto(basePage)(url, {
|
|
@@ -189,87 +171,41 @@ function retryWithProxy(page, options) {
|
|
|
189
171
|
throw new AluviaError("Your Aluvia account has no remaining balance. Please top up at https://dashboard.aluvia.io/ to continue using proxies.", AluviaErrorCode.InsufficientBalance);
|
|
190
172
|
}
|
|
191
173
|
}
|
|
192
|
-
const proxy = await (proxyProvider?.get() ?? getAluviaProxy()).catch((err) => {
|
|
193
|
-
lastErr = err;
|
|
194
|
-
return undefined;
|
|
195
|
-
});
|
|
196
|
-
if (!proxy) {
|
|
197
|
-
throw new AluviaError("Failed to obtain a proxy for retry attempts. Check your balance and proxy pool at https://dashboard.aluvia.io/.", AluviaErrorCode.ProxyFetchFailed);
|
|
198
|
-
}
|
|
199
|
-
else {
|
|
200
|
-
await onProxyLoaded?.(proxy);
|
|
201
|
-
}
|
|
202
|
-
// If dynamic proxy supplied, switch upstream & retry on same page without relaunch.
|
|
203
|
-
if (dynamicProxy) {
|
|
204
|
-
await dynamicProxy.setUpstream(proxy);
|
|
205
|
-
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
206
|
-
if (backoffMs > 0) {
|
|
207
|
-
const delay = backoffDelay(backoffMs, attempt - 1);
|
|
208
|
-
await new Promise((r) => setTimeout(r, delay));
|
|
209
|
-
}
|
|
210
|
-
await onRetry?.(attempt, maxRetries, lastErr);
|
|
211
|
-
try {
|
|
212
|
-
const response = await getRawGoto(basePage)(url, {
|
|
213
|
-
...(gotoOptions ?? {}),
|
|
214
|
-
timeout: gotoOptions?.timeout ?? DEFAULT_GOTO_TIMEOUT_MS,
|
|
215
|
-
waitUntil: gotoOptions?.waitUntil ?? "domcontentloaded",
|
|
216
|
-
});
|
|
217
|
-
return { response: response ?? null, page: basePage };
|
|
218
|
-
}
|
|
219
|
-
catch (err) {
|
|
220
|
-
lastErr = err;
|
|
221
|
-
if (!isRetryable(err))
|
|
222
|
-
break; // stop early on non-retryable error
|
|
223
|
-
continue; // next attempt
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
if (lastErr instanceof Error)
|
|
227
|
-
throw lastErr;
|
|
228
|
-
throw new Error(lastErr ? String(lastErr) : "Navigation failed");
|
|
229
|
-
}
|
|
230
|
-
// Original relaunch path if no dynamic proxy provided
|
|
231
|
-
// Retries with proxy
|
|
232
174
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
175
|
+
const proxy = await (proxyProvider?.get() ?? getAluviaProxy()).catch((err) => {
|
|
176
|
+
lastErr = err;
|
|
177
|
+
return undefined;
|
|
178
|
+
});
|
|
179
|
+
if (!proxy) {
|
|
180
|
+
throw new AluviaError("Failed to obtain a proxy for retry attempts. Check your balance and proxy pool at https://dashboard.aluvia.io/.", AluviaErrorCode.ProxyFetchFailed);
|
|
181
|
+
}
|
|
182
|
+
else {
|
|
183
|
+
await onProxyLoaded?.(proxy);
|
|
184
|
+
}
|
|
185
|
+
// switch upstream & retry on same page without relaunch.
|
|
186
|
+
await dynamicProxy.setUpstream(proxy);
|
|
233
187
|
if (backoffMs > 0) {
|
|
234
188
|
const delay = backoffDelay(backoffMs, attempt - 1);
|
|
235
|
-
await new Promise((
|
|
189
|
+
await new Promise((r) => setTimeout(r, delay));
|
|
236
190
|
}
|
|
237
191
|
await onRetry?.(attempt, maxRetries, lastErr);
|
|
238
192
|
try {
|
|
239
|
-
const
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
});
|
|
246
|
-
// non-fatal readiness gate
|
|
247
|
-
try {
|
|
248
|
-
await newPage.waitForFunction(() => typeof document !== "undefined" && !!document.title?.trim(), { timeout: DEFAULT_GOTO_TIMEOUT_MS });
|
|
249
|
-
}
|
|
250
|
-
catch { }
|
|
251
|
-
return {
|
|
252
|
-
response: response ?? null,
|
|
253
|
-
page: newPage,
|
|
254
|
-
};
|
|
255
|
-
}
|
|
256
|
-
catch (err) {
|
|
257
|
-
// navigation on the new page failed — carry this page forward
|
|
258
|
-
basePage = newPage;
|
|
259
|
-
lastErr = err;
|
|
260
|
-
// next loop iteration will close this browser (since we pass basePage)
|
|
261
|
-
continue;
|
|
262
|
-
}
|
|
193
|
+
const response = await getRawGoto(basePage)(url, {
|
|
194
|
+
...(gotoOptions ?? {}),
|
|
195
|
+
timeout: gotoOptions?.timeout ?? DEFAULT_GOTO_TIMEOUT_MS,
|
|
196
|
+
waitUntil: gotoOptions?.waitUntil ?? "domcontentloaded",
|
|
197
|
+
});
|
|
198
|
+
return { response: response ?? null, page: basePage };
|
|
263
199
|
}
|
|
264
200
|
catch (err) {
|
|
265
|
-
// relaunch itself failed (no new page created)
|
|
266
201
|
lastErr = err;
|
|
267
|
-
|
|
202
|
+
if (!isRetryable(err))
|
|
203
|
+
break; // stop early on non-retryable error
|
|
204
|
+
continue; // next attempt
|
|
268
205
|
}
|
|
269
206
|
}
|
|
270
|
-
if (lastErr instanceof Error)
|
|
207
|
+
if (lastErr instanceof Error)
|
|
271
208
|
throw lastErr;
|
|
272
|
-
}
|
|
273
209
|
throw new Error(lastErr ? String(lastErr) : "Navigation failed");
|
|
274
210
|
};
|
|
275
211
|
return run();
|
package/dist/esm/src/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Server as ProxyChainServer } from "proxy-chain";
|
|
2
2
|
const DEFAULT_GOTO_TIMEOUT_MS = 15000;
|
|
3
|
-
const ENV_MAX_RETRIES = Math.max(0, parseInt(process.env.ALUVIA_MAX_RETRIES || "
|
|
3
|
+
const ENV_MAX_RETRIES = Math.max(0, parseInt(process.env.ALUVIA_MAX_RETRIES || "2", 10)); // prettier-ignore
|
|
4
4
|
const ENV_BACKOFF_MS = Math.max(0, parseInt(process.env.ALUVIA_BACKOFF_MS || "300", 10)); // prettier-ignore
|
|
5
5
|
const ENV_RETRY_ON = (process.env.ALUVIA_RETRY_ON ?? "ECONNRESET,ETIMEDOUT,net::ERR,Timeout")
|
|
6
6
|
.split(",")
|
|
@@ -17,6 +17,7 @@ var AluviaErrorCode;
|
|
|
17
17
|
AluviaErrorCode["ProxyFetchFailed"] = "ALUVIA_PROXY_FETCH_FAILED";
|
|
18
18
|
AluviaErrorCode["InsufficientBalance"] = "ALUVIA_INSUFFICIENT_BALANCE";
|
|
19
19
|
AluviaErrorCode["BalanceFetchFailed"] = "ALUVIA_BALANCE_FETCH_FAILED";
|
|
20
|
+
AluviaErrorCode["NoDynamicProxy"] = "ALUVIA_NO_DYNAMIC_PROXY";
|
|
20
21
|
})(AluviaErrorCode || (AluviaErrorCode = {}));
|
|
21
22
|
export class AluviaError extends Error {
|
|
22
23
|
constructor(message, code) {
|
|
@@ -27,9 +28,9 @@ export class AluviaError extends Error {
|
|
|
27
28
|
}
|
|
28
29
|
let aluviaClient; // lazy-loaded Aluvia client instance
|
|
29
30
|
async function getAluviaProxy() {
|
|
30
|
-
const apiKey = process.env.
|
|
31
|
+
const apiKey = process.env.ALUVIA_TOKEN || "";
|
|
31
32
|
if (!apiKey) {
|
|
32
|
-
throw new AluviaError("Missing
|
|
33
|
+
throw new AluviaError("Missing ALUVIA_TOKEN environment variable.", AluviaErrorCode.NoApiKey);
|
|
33
34
|
}
|
|
34
35
|
if (!aluviaClient) {
|
|
35
36
|
// Dynamic import to play nicely with test mocks (avoids top-level evaluation before vi.mock)
|
|
@@ -41,16 +42,17 @@ async function getAluviaProxy() {
|
|
|
41
42
|
if (!proxy) {
|
|
42
43
|
throw new AluviaError("Failed to obtain a proxy for retry attempts. Check your balance and proxy pool at https://dashboard.aluvia.io/.", AluviaErrorCode.NoProxy);
|
|
43
44
|
}
|
|
45
|
+
const sessionId = generateSessionId();
|
|
44
46
|
return {
|
|
45
47
|
server: `http://${proxy.host}:${proxy.httpPort}`,
|
|
46
|
-
username: proxy.username
|
|
48
|
+
username: `${proxy.username}-session-${sessionId}`,
|
|
47
49
|
password: proxy.password,
|
|
48
50
|
};
|
|
49
51
|
}
|
|
50
52
|
async function getAluviaBalance() {
|
|
51
|
-
const apiKey = process.env.
|
|
53
|
+
const apiKey = process.env.ALUVIA_TOKEN || "";
|
|
52
54
|
if (!apiKey) {
|
|
53
|
-
throw new AluviaError("Missing
|
|
55
|
+
throw new AluviaError("Missing ALUVIA_TOKEN environment variable.", AluviaErrorCode.NoApiKey);
|
|
54
56
|
}
|
|
55
57
|
const response = await fetch("https://api.aluvia.io/account/status", {
|
|
56
58
|
headers: {
|
|
@@ -80,47 +82,16 @@ function compileRetryable(patterns = DEFAULT_RETRY_PATTERNS) {
|
|
|
80
82
|
: msg.includes(p) || code.includes(p) || name.includes(p));
|
|
81
83
|
};
|
|
82
84
|
}
|
|
83
|
-
function
|
|
84
|
-
|
|
85
|
-
const browserType = browser?.browserType?.();
|
|
86
|
-
if (!browserType) {
|
|
87
|
-
throw new Error("Cannot infer BrowserType from page");
|
|
88
|
-
}
|
|
89
|
-
return browserType;
|
|
90
|
-
}
|
|
91
|
-
async function inferContextDefaults(page) {
|
|
92
|
-
const context = page.context();
|
|
93
|
-
const options = context._options;
|
|
94
|
-
return options ?? {};
|
|
95
|
-
}
|
|
96
|
-
function inferLaunchDefaults(page) {
|
|
97
|
-
const browser = page.context().browser();
|
|
98
|
-
const options = browser._options;
|
|
99
|
-
return options ?? {};
|
|
100
|
-
}
|
|
101
|
-
async function relaunchWithProxy(proxy, oldPage, closeOldBrowser = true) {
|
|
102
|
-
const browserType = inferBrowserTypeFromPage(oldPage);
|
|
103
|
-
const launchDefaults = inferLaunchDefaults(oldPage);
|
|
104
|
-
const contextDefaults = await inferContextDefaults(oldPage);
|
|
105
|
-
if (closeOldBrowser) {
|
|
106
|
-
const oldBrowser = oldPage.context().browser();
|
|
107
|
-
try {
|
|
108
|
-
await oldBrowser?.close();
|
|
109
|
-
}
|
|
110
|
-
catch { }
|
|
111
|
-
}
|
|
112
|
-
const retryLaunch = {
|
|
113
|
-
...launchDefaults,
|
|
114
|
-
proxy,
|
|
115
|
-
};
|
|
116
|
-
const browser = await browserType.launch(retryLaunch);
|
|
117
|
-
const context = await browser.newContext(contextDefaults);
|
|
118
|
-
const page = await context.newPage();
|
|
119
|
-
return { page };
|
|
85
|
+
function generateSessionId() {
|
|
86
|
+
return Math.random().toString(36).substring(2, 10);
|
|
120
87
|
}
|
|
121
88
|
const GOTO_ORIGINAL = Symbol.for("aluvia.gotoOriginal");
|
|
122
|
-
|
|
123
|
-
|
|
89
|
+
const CONTEXT_LISTENER_ATTACHED = new WeakSet();
|
|
90
|
+
export function agentConnect(page, options) {
|
|
91
|
+
const { dynamicProxy, maxRetries = ENV_MAX_RETRIES, backoffMs = ENV_BACKOFF_MS, retryOn = DEFAULT_RETRY_PATTERNS, proxyProvider, onRetry, onProxyLoaded, } = options ?? {};
|
|
92
|
+
if (!dynamicProxy) {
|
|
93
|
+
throw new AluviaError("No dynamic proxy supplied to agentConnect", AluviaErrorCode.NoDynamicProxy);
|
|
94
|
+
}
|
|
124
95
|
const isRetryable = compileRetryable(retryOn);
|
|
125
96
|
/** Prefer unpatched goto to avoid recursion */
|
|
126
97
|
const getRawGoto = (p) => (p[GOTO_ORIGINAL]?.bind(p) ?? p.goto.bind(p));
|
|
@@ -129,6 +100,17 @@ export function retryWithProxy(page, options) {
|
|
|
129
100
|
const run = async () => {
|
|
130
101
|
let basePage = page;
|
|
131
102
|
let lastErr;
|
|
103
|
+
// One-time attach context close listener to shut down dynamic proxy
|
|
104
|
+
if (dynamicProxy) {
|
|
105
|
+
const ctx = basePage.context();
|
|
106
|
+
if (!CONTEXT_LISTENER_ATTACHED.has(ctx)) {
|
|
107
|
+
ctx.on('close', async () => { try {
|
|
108
|
+
await dynamicProxy.close();
|
|
109
|
+
}
|
|
110
|
+
catch { } });
|
|
111
|
+
CONTEXT_LISTENER_ATTACHED.add(ctx);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
132
114
|
// First attempt without proxy
|
|
133
115
|
try {
|
|
134
116
|
const response = await getRawGoto(basePage)(url, {
|
|
@@ -150,87 +132,41 @@ export function retryWithProxy(page, options) {
|
|
|
150
132
|
throw new AluviaError("Your Aluvia account has no remaining balance. Please top up at https://dashboard.aluvia.io/ to continue using proxies.", AluviaErrorCode.InsufficientBalance);
|
|
151
133
|
}
|
|
152
134
|
}
|
|
153
|
-
const proxy = await (proxyProvider?.get() ?? getAluviaProxy()).catch((err) => {
|
|
154
|
-
lastErr = err;
|
|
155
|
-
return undefined;
|
|
156
|
-
});
|
|
157
|
-
if (!proxy) {
|
|
158
|
-
throw new AluviaError("Failed to obtain a proxy for retry attempts. Check your balance and proxy pool at https://dashboard.aluvia.io/.", AluviaErrorCode.ProxyFetchFailed);
|
|
159
|
-
}
|
|
160
|
-
else {
|
|
161
|
-
await onProxyLoaded?.(proxy);
|
|
162
|
-
}
|
|
163
|
-
// If dynamic proxy supplied, switch upstream & retry on same page without relaunch.
|
|
164
|
-
if (dynamicProxy) {
|
|
165
|
-
await dynamicProxy.setUpstream(proxy);
|
|
166
|
-
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
167
|
-
if (backoffMs > 0) {
|
|
168
|
-
const delay = backoffDelay(backoffMs, attempt - 1);
|
|
169
|
-
await new Promise((r) => setTimeout(r, delay));
|
|
170
|
-
}
|
|
171
|
-
await onRetry?.(attempt, maxRetries, lastErr);
|
|
172
|
-
try {
|
|
173
|
-
const response = await getRawGoto(basePage)(url, {
|
|
174
|
-
...(gotoOptions ?? {}),
|
|
175
|
-
timeout: gotoOptions?.timeout ?? DEFAULT_GOTO_TIMEOUT_MS,
|
|
176
|
-
waitUntil: gotoOptions?.waitUntil ?? "domcontentloaded",
|
|
177
|
-
});
|
|
178
|
-
return { response: response ?? null, page: basePage };
|
|
179
|
-
}
|
|
180
|
-
catch (err) {
|
|
181
|
-
lastErr = err;
|
|
182
|
-
if (!isRetryable(err))
|
|
183
|
-
break; // stop early on non-retryable error
|
|
184
|
-
continue; // next attempt
|
|
185
|
-
}
|
|
186
|
-
}
|
|
187
|
-
if (lastErr instanceof Error)
|
|
188
|
-
throw lastErr;
|
|
189
|
-
throw new Error(lastErr ? String(lastErr) : "Navigation failed");
|
|
190
|
-
}
|
|
191
|
-
// Original relaunch path if no dynamic proxy provided
|
|
192
|
-
// Retries with proxy
|
|
193
135
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
136
|
+
const proxy = await (proxyProvider?.get() ?? getAluviaProxy()).catch((err) => {
|
|
137
|
+
lastErr = err;
|
|
138
|
+
return undefined;
|
|
139
|
+
});
|
|
140
|
+
if (!proxy) {
|
|
141
|
+
throw new AluviaError("Failed to obtain a proxy for retry attempts. Check your balance and proxy pool at https://dashboard.aluvia.io/.", AluviaErrorCode.ProxyFetchFailed);
|
|
142
|
+
}
|
|
143
|
+
else {
|
|
144
|
+
await onProxyLoaded?.(proxy);
|
|
145
|
+
}
|
|
146
|
+
// switch upstream & retry on same page without relaunch.
|
|
147
|
+
await dynamicProxy.setUpstream(proxy);
|
|
194
148
|
if (backoffMs > 0) {
|
|
195
149
|
const delay = backoffDelay(backoffMs, attempt - 1);
|
|
196
|
-
await new Promise((
|
|
150
|
+
await new Promise((r) => setTimeout(r, delay));
|
|
197
151
|
}
|
|
198
152
|
await onRetry?.(attempt, maxRetries, lastErr);
|
|
199
153
|
try {
|
|
200
|
-
const
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
});
|
|
207
|
-
// non-fatal readiness gate
|
|
208
|
-
try {
|
|
209
|
-
await newPage.waitForFunction(() => typeof document !== "undefined" && !!document.title?.trim(), { timeout: DEFAULT_GOTO_TIMEOUT_MS });
|
|
210
|
-
}
|
|
211
|
-
catch { }
|
|
212
|
-
return {
|
|
213
|
-
response: response ?? null,
|
|
214
|
-
page: newPage,
|
|
215
|
-
};
|
|
216
|
-
}
|
|
217
|
-
catch (err) {
|
|
218
|
-
// navigation on the new page failed — carry this page forward
|
|
219
|
-
basePage = newPage;
|
|
220
|
-
lastErr = err;
|
|
221
|
-
// next loop iteration will close this browser (since we pass basePage)
|
|
222
|
-
continue;
|
|
223
|
-
}
|
|
154
|
+
const response = await getRawGoto(basePage)(url, {
|
|
155
|
+
...(gotoOptions ?? {}),
|
|
156
|
+
timeout: gotoOptions?.timeout ?? DEFAULT_GOTO_TIMEOUT_MS,
|
|
157
|
+
waitUntil: gotoOptions?.waitUntil ?? "domcontentloaded",
|
|
158
|
+
});
|
|
159
|
+
return { response: response ?? null, page: basePage };
|
|
224
160
|
}
|
|
225
161
|
catch (err) {
|
|
226
|
-
// relaunch itself failed (no new page created)
|
|
227
162
|
lastErr = err;
|
|
228
|
-
|
|
163
|
+
if (!isRetryable(err))
|
|
164
|
+
break; // stop early on non-retryable error
|
|
165
|
+
continue; // next attempt
|
|
229
166
|
}
|
|
230
167
|
}
|
|
231
|
-
if (lastErr instanceof Error)
|
|
168
|
+
if (lastErr instanceof Error)
|
|
232
169
|
throw lastErr;
|
|
233
|
-
}
|
|
234
170
|
throw new Error(lastErr ? String(lastErr) : "Navigation failed");
|
|
235
171
|
};
|
|
236
172
|
return run();
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { Page, Response } from "playwright";
|
|
2
2
|
export type RetryPattern = string | RegExp;
|
|
3
3
|
type GoToOptions = NonNullable<Parameters<Page["goto"]>[1]>;
|
|
4
|
-
export interface
|
|
4
|
+
export interface AgentConnectRunner {
|
|
5
5
|
goto(url: string, options?: GoToOptions): Promise<{
|
|
6
6
|
response: Response | null;
|
|
7
7
|
page: Page;
|
|
@@ -20,13 +20,22 @@ declare enum AluviaErrorCode {
|
|
|
20
20
|
NoProxy = "ALUVIA_NO_PROXIES",
|
|
21
21
|
ProxyFetchFailed = "ALUVIA_PROXY_FETCH_FAILED",
|
|
22
22
|
InsufficientBalance = "ALUVIA_INSUFFICIENT_BALANCE",
|
|
23
|
-
BalanceFetchFailed = "ALUVIA_BALANCE_FETCH_FAILED"
|
|
23
|
+
BalanceFetchFailed = "ALUVIA_BALANCE_FETCH_FAILED",
|
|
24
|
+
NoDynamicProxy = "ALUVIA_NO_DYNAMIC_PROXY"
|
|
24
25
|
}
|
|
25
26
|
export declare class AluviaError extends Error {
|
|
26
27
|
code?: AluviaErrorCode;
|
|
27
28
|
constructor(message: string, code?: AluviaErrorCode);
|
|
28
29
|
}
|
|
29
|
-
export interface
|
|
30
|
+
export interface AgentConnectOptions {
|
|
31
|
+
/**
|
|
32
|
+
* Dynamic proxy. Retries will switch upstream proxy via this local proxy.
|
|
33
|
+
*
|
|
34
|
+
* To use: const dyn = await startDynamicProxy();
|
|
35
|
+
* chromium.launch({ proxy: { server: dyn.url } })
|
|
36
|
+
* Then pass { dynamicProxy: dyn } to agentConnect().
|
|
37
|
+
*/
|
|
38
|
+
dynamicProxy: DynamicProxy;
|
|
30
39
|
/**
|
|
31
40
|
* Number of retry attempts after the first failed navigation.
|
|
32
41
|
*
|
|
@@ -34,7 +43,7 @@ export interface RetryWithProxyOptions {
|
|
|
34
43
|
* If it fails with a retryable error (as defined by `retryOn`),
|
|
35
44
|
* the helper will fetch a new proxy and relaunch the browser.
|
|
36
45
|
*
|
|
37
|
-
* @default process.env.ALUVIA_MAX_RETRIES ||
|
|
46
|
+
* @default process.env.ALUVIA_MAX_RETRIES || 2
|
|
38
47
|
* @example
|
|
39
48
|
* // Try up to 3 proxy relaunches after the first failure
|
|
40
49
|
* { maxRetries: 3 }
|
|
@@ -65,25 +74,12 @@ export interface RetryWithProxyOptions {
|
|
|
65
74
|
* { retryOn: ["ECONNRESET", /403/] }
|
|
66
75
|
*/
|
|
67
76
|
retryOn?: RetryPattern[];
|
|
68
|
-
/**
|
|
69
|
-
* Whether to close the old browser instance when relaunching with a new proxy.
|
|
70
|
-
*
|
|
71
|
-
* Set to `true` (default) to prevent multiple browsers from staying open,
|
|
72
|
-
* which is safer for most workflows. Set to `false` if you manage browser
|
|
73
|
-
* lifecycles manually or reuse a shared browser across tasks.
|
|
74
|
-
*
|
|
75
|
-
* @default true
|
|
76
|
-
* @example
|
|
77
|
-
* // Keep old browser open (you must close it yourself)
|
|
78
|
-
* { closeOldBrowser: false }
|
|
79
|
-
*/
|
|
80
|
-
closeOldBrowser?: boolean;
|
|
81
77
|
/**
|
|
82
78
|
* Optional custom proxy provider used to fetch proxy credentials.
|
|
83
79
|
*
|
|
84
|
-
* By default, `
|
|
80
|
+
* By default, `agentConnect` automatically uses the Aluvia API
|
|
85
81
|
* via the `aluvia-ts-sdk` and reads the API key from
|
|
86
|
-
* `process.env.
|
|
82
|
+
* `process.env.ALUVIA_TOKEN`.
|
|
87
83
|
*
|
|
88
84
|
* Supplying your own `proxyProvider` allows you to integrate with
|
|
89
85
|
* any proxy rotation service, database, or in-house pool instead.
|
|
@@ -92,10 +88,10 @@ export interface RetryWithProxyOptions {
|
|
|
92
88
|
* `Promise<ProxySettings>` object with `server`, and optionally
|
|
93
89
|
* `username` and `password` fields.
|
|
94
90
|
*
|
|
95
|
-
* @default Uses the built-in Aluvia client with `process.env.
|
|
91
|
+
* @default Uses the built-in Aluvia client with `process.env.ALUVIA_TOKEN`
|
|
96
92
|
* @example
|
|
97
93
|
* ```ts
|
|
98
|
-
* import {
|
|
94
|
+
* import { agentConnect } from "agent-connect";
|
|
99
95
|
*
|
|
100
96
|
* // Custom proxy provider example
|
|
101
97
|
* const myProxyProvider = {
|
|
@@ -109,7 +105,7 @@ export interface RetryWithProxyOptions {
|
|
|
109
105
|
* },
|
|
110
106
|
* };
|
|
111
107
|
*
|
|
112
|
-
* const { response, page } = await
|
|
108
|
+
* const { response, page } = await agentConnect(page, {
|
|
113
109
|
* proxyProvider: myProxyProvider,
|
|
114
110
|
* maxRetries: 3,
|
|
115
111
|
* });
|
|
@@ -130,17 +126,8 @@ export interface RetryWithProxyOptions {
|
|
|
130
126
|
* @param proxy The proxy settings that were fetched or provided
|
|
131
127
|
*/
|
|
132
128
|
onProxyLoaded?: (proxy: ProxySettings) => void | Promise<void>;
|
|
133
|
-
/**
|
|
134
|
-
* Optional dynamic proxy. If provided, retries will switch upstream proxy
|
|
135
|
-
* via this local proxy instead of relaunching the browser.
|
|
136
|
-
*
|
|
137
|
-
* To use: const dyn = await startDynamicProxy();
|
|
138
|
-
* chromium.launch({ proxy: { server: dyn.url } })
|
|
139
|
-
* Then pass { dynamicProxy: dyn } to retryWithProxy().
|
|
140
|
-
*/
|
|
141
|
-
dynamicProxy?: DynamicProxy;
|
|
142
129
|
}
|
|
143
|
-
export declare function
|
|
130
|
+
export declare function agentConnect(page: Page, options?: AgentConnectOptions): AgentConnectRunner;
|
|
144
131
|
/**
|
|
145
132
|
* Starts a local proxy-chain server which can have its upstream changed at runtime
|
|
146
133
|
* without relaunching the browser. Launch Playwright with { proxy: { server: dynamic.url } }.
|
package/package.json
CHANGED