onebrowser 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -33
- package/package.json +2 -2
- package/src/account.js +139 -0
- package/src/cli.js +29 -24
- package/src/cost.js +8 -6
- package/src/credentials.js +17 -12
- package/src/index.js +54 -186
- package/src/ledger.js +74 -63
- package/src/options.js +30 -0
- package/src/providers/anchor.js +1 -0
- package/src/providers/index.js +22 -25
- package/src/session.js +27 -38
package/src/index.js
CHANGED
|
@@ -1,240 +1,108 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { Account } from './account.js';
|
|
2
|
+
import { readApiKeys } from './credentials.js';
|
|
3
3
|
import { QuotaError } from './errors.js';
|
|
4
4
|
import { defaultLedgerPath, Ledger } from './ledger.js';
|
|
5
|
-
import { ADAPTERS, checkAdapter,
|
|
6
|
-
import { KEEP_ALIVE_MS, Session } from './session.js';
|
|
5
|
+
import { ADAPTERS, checkAdapter, fullSettings } from './providers/index.js';
|
|
7
6
|
|
|
8
7
|
export { apiKeyEnv } from './credentials.js';
|
|
8
|
+
export { sessionOptions } from './options.js';
|
|
9
9
|
export { QuotaError };
|
|
10
10
|
|
|
11
|
-
/** Session options when openSession() is not given them */
|
|
12
|
-
const SESSION_DEFAULTS = {
|
|
13
|
-
proxyCountry: null, // null: the provider's default egress
|
|
14
|
-
idleTimeoutSec: 60, // a session whose client is gone (its process was killed) ends this long after at the latest
|
|
15
|
-
maxDurationMs: 15 * 60_000,
|
|
16
|
-
};
|
|
17
|
-
|
|
18
|
-
/**
|
|
19
|
-
* The session options of openSession() with the defaults filled in (SESSION_DEFAULTS); throws when one is unknown
|
|
20
|
-
* or out of range, so that a program can check its configuration before it opens any session:
|
|
21
|
-
* proxyCountry country code: the provider's proxy there; null: its default egress
|
|
22
|
-
* idleTimeoutSec a session whose client is gone ends this long after at the latest; at least twice the
|
|
23
|
-
* keep-alive interval (session.js)
|
|
24
|
-
* maxDurationMs the session ends this long after it was opened at the latest
|
|
25
|
-
*/
|
|
26
|
-
export function sessionOptions(options = {}) {
|
|
27
|
-
const unknown = Object.keys(options).filter((k) => !Object.hasOwn(SESSION_DEFAULTS, k));
|
|
28
|
-
if (unknown.length) throw new Error(`unknown session option ${unknown.join(', ')} (expected ${Object.keys(SESSION_DEFAULTS).join(', ')})`);
|
|
29
|
-
const o = { ...SESSION_DEFAULTS, ...options };
|
|
30
|
-
if (!(Number.isFinite(o.idleTimeoutSec) && o.idleTimeoutSec * 1000 >= 2 * KEEP_ALIVE_MS)) {
|
|
31
|
-
throw new Error(`idleTimeoutSec must be at least ${(2 * KEEP_ALIVE_MS) / 1000} (twice the keep-alive interval), not ${o.idleTimeoutSec}`);
|
|
32
|
-
}
|
|
33
|
-
if (!(Number.isFinite(o.maxDurationMs) && o.maxDurationMs > 0)) throw new Error(`maxDurationMs must be a number > 0, not ${o.maxDurationMs}`);
|
|
34
|
-
return o;
|
|
35
|
-
}
|
|
36
|
-
|
|
37
11
|
/**
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
/**
|
|
45
|
-
* Remote browsers from several providers behind one interface: a session is asked for by provider name, the way
|
|
46
|
-
* an LLM request names its model. Every session is recorded in a ledger (ledger.js), which the usage of each
|
|
47
|
-
* provider account and, for providers that cannot report it themselves, its quota are read from.
|
|
48
|
-
*
|
|
49
|
-
* What is left of an account's quota is what its provider reads, less what the sessions it has not billed yet
|
|
50
|
-
* cost (quota()): the ledger knows those, and each provider's price (cost.js) what they cost. So a program can
|
|
51
|
-
* tell, before it opens a session, whether the account can pay for it.
|
|
12
|
+
* Remote browsers from several providers behind one interface. A provider is an adapter (providers/index.js);
|
|
13
|
+
* an account is one API key at a provider (account.js), which sessions are opened with. Every session is
|
|
14
|
+
* recorded in a ledger (ledger.js) shared by the programs on the machine, so that each account's usage, and
|
|
15
|
+
* what its quota has left at any moment, are known. Which account to use, and when to stop using one, are the
|
|
16
|
+
* caller's decisions.
|
|
52
17
|
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
* new OneBrowser({ client, ledgerPath, providers, adapters })
|
|
57
|
-
* client the program using it, recorded with each session (several programs may share one ledger)
|
|
58
|
-
* ledgerPath SQLite file of the ledger (default: defaultLedgerPath() in ledger.js; ':memory:' for tests),
|
|
59
|
-
* opened when first needed
|
|
60
|
-
* providers settings per provider name: apiKey (else from the environment, see credentials.js), and the
|
|
61
|
-
* adapter's own settings on top of its defaults (providers/<name>.js)
|
|
62
|
-
* adapters further adapters by provider name, or replacements of built-in ones (providers/index.js)
|
|
18
|
+
* client: the program using it, recorded with each session; ledgerPath: the ledger's SQLite file, opened when
|
|
19
|
+
* first needed; providers: settings per provider name on top of its adapter's defaults, apiKeys among them
|
|
20
|
+
* (else credentials.js); adapters: further or replacement adapters by provider name.
|
|
63
21
|
*/
|
|
64
22
|
export class OneBrowser {
|
|
65
|
-
#providers =
|
|
23
|
+
#providers = []; // names, in the adapters' order
|
|
24
|
+
#accounts = [];
|
|
66
25
|
#ledger = null;
|
|
67
|
-
#open = new Set();
|
|
68
|
-
#readings = new Map(); // provider → its last quota reading (#reading)
|
|
69
26
|
|
|
70
27
|
constructor({ client, ledgerPath = defaultLedgerPath(), providers = {}, adapters = {} }) {
|
|
71
28
|
if (!client?.trim()) throw new Error('onebrowser: client must name the program using it');
|
|
72
29
|
this.client = client;
|
|
73
30
|
this.ledgerPath = ledgerPath;
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
const { apiKey, ...given } = providers[name] ?? {};
|
|
77
|
-
const settings = { ...adapter.defaults, ...given };
|
|
78
|
-
checkSettings(name, settings);
|
|
79
|
-
this.#providers.set(name, { adapter, settings, apiKey });
|
|
80
|
-
}
|
|
81
|
-
const unknown = Object.keys(providers).filter((name) => !this.#providers.has(name));
|
|
31
|
+
const all = { ...ADAPTERS, ...adapters };
|
|
32
|
+
const unknown = Object.keys(providers).filter((name) => !all[name]);
|
|
82
33
|
if (unknown.length) throw new Error(`onebrowser: no adapter for provider ${unknown.join(', ')}`);
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
return (this.#ledger ??= new Ledger(this.ledgerPath));
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
/** The adapter, settings and the API key given of `provider`; throws when it is unknown */
|
|
95
|
-
#known(provider) {
|
|
96
|
-
const p = this.#providers.get(provider);
|
|
97
|
-
if (!p) throw new Error(`unknown provider ${provider} (expected ${this.providers.join(', ')})`);
|
|
98
|
-
return p;
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
/** The adapter, settings and API key of `provider`; throws when it is unknown or has no API key */
|
|
102
|
-
#ready(provider) {
|
|
103
|
-
const p = this.#known(provider);
|
|
104
|
-
return { ...p, apiKey: readApiKey(provider, p.apiKey) };
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
/** Throws when sessions cannot be opened with `provider`: unknown, or no API key */
|
|
108
|
-
checkReady(provider) {
|
|
109
|
-
this.#ready(provider);
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
/**
|
|
113
|
-
* A new session with `provider`, recorded in the ledger once the provider has created it; options: see
|
|
114
|
-
* sessionOptions(). Throws when the provider does not create one; QuotaError when it refuses for the account's
|
|
115
|
-
* quota.
|
|
116
|
-
*/
|
|
117
|
-
async openSession({ provider, ...options }) {
|
|
118
|
-
const { adapter, settings, apiKey } = this.#ready(provider);
|
|
119
|
-
const o = sessionOptions(options);
|
|
120
|
-
const openedAt = Date.now(); // the provider bills the session from some time after this
|
|
121
|
-
let remote;
|
|
122
|
-
try {
|
|
123
|
-
remote = await adapter.createSession(settings, apiKey, o);
|
|
124
|
-
} catch (e) {
|
|
125
|
-
if (e instanceof QuotaError) throw e;
|
|
126
|
-
throw new Error(`failed to create a ${provider} session: ${e.message}`);
|
|
127
|
-
}
|
|
128
|
-
const session = new Session({
|
|
129
|
-
provider, remote, openedAt, maxDurationMs: o.maxDurationMs,
|
|
130
|
-
release: async (s) => {
|
|
131
|
-
this.#open.delete(s);
|
|
132
|
-
this.#store.closed(s);
|
|
133
|
-
await adapter.releaseSession(settings, apiKey, s);
|
|
134
|
-
},
|
|
135
|
-
});
|
|
136
|
-
try {
|
|
137
|
-
this.#store.opened({ provider, id: session.id, client: this.client, openedAt, endsBy: openedAt + sessionMsAtMost(o) });
|
|
138
|
-
} catch (e) {
|
|
139
|
-
await session.close().catch(() => {}); // a session the ledger has no record of is not handed out
|
|
140
|
-
throw e;
|
|
34
|
+
for (const [provider, adapter] of Object.entries(all)) {
|
|
35
|
+
checkAdapter(provider, adapter);
|
|
36
|
+
const { apiKeys, ...given } = providers[provider] ?? {};
|
|
37
|
+
const settings = fullSettings(provider, { ...adapter.defaults, ...given });
|
|
38
|
+
this.#providers.push(provider);
|
|
39
|
+
for (const apiKey of readApiKeys(provider, apiKeys)) {
|
|
40
|
+
this.#accounts.push(new Account({ provider, adapter, settings, apiKey, client, ledger: () => this.#store }));
|
|
41
|
+
}
|
|
141
42
|
}
|
|
142
|
-
this.#open.add(session);
|
|
143
|
-
return session;
|
|
144
43
|
}
|
|
145
44
|
|
|
146
45
|
/**
|
|
147
|
-
*
|
|
148
|
-
*
|
|
149
|
-
* reading may not show yet cost, each at the provider's price (cost.js). Those are the sessions that had not
|
|
150
|
-
* ended billingDelayMs before the reading: open ones as if they lasted as long as they can, ended ones as long
|
|
151
|
-
* as they lasted. renewsAt (ms) when the quota renews by itself, null when only a top-up renews it.
|
|
152
|
-
*
|
|
153
|
-
* The provider is read anew unless a reading started at most maxAgeMs ago, or still in progress, can be used
|
|
154
|
-
* (a failed one too: it throws again); what is unbilled is worked out from the ledger every time, so it includes
|
|
155
|
-
* the sessions opened since, by any program sharing the ledger. Throws when the provider cannot be read.
|
|
46
|
+
* The ledger, opened on first use. Sessions recorded before accounts were (onebrowser 0.5 and earlier) are
|
|
47
|
+
* the provider's only account's, when it has one here; with several, no one knows which they belong to.
|
|
156
48
|
*/
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
49
|
+
get #store() {
|
|
50
|
+
if (!this.#ledger) {
|
|
51
|
+
this.#ledger = new Ledger(this.ledgerPath);
|
|
52
|
+
for (const [provider, accounts] of Map.groupBy(this.#accounts, (a) => a.provider)) {
|
|
53
|
+
if (accounts.length === 1) this.#ledger.claim(provider, accounts[0].id);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
return this.#ledger;
|
|
164
57
|
}
|
|
165
58
|
|
|
166
|
-
/**
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
* error is the answer until maxAgeMs have passed.
|
|
170
|
-
*/
|
|
171
|
-
#reading(provider, maxAgeMs) {
|
|
172
|
-
const last = this.#readings.get(provider);
|
|
173
|
-
if (last && (!last.done || Date.now() - last.startedAt <= maxAgeMs)) return last;
|
|
174
|
-
const { adapter, settings, apiKey } = this.#ready(provider);
|
|
175
|
-
const reading = { startedAt: Date.now(), done: false };
|
|
176
|
-
reading.answer = adapter
|
|
177
|
-
.quota(settings, apiKey, this.#store.history(provider))
|
|
178
|
-
.then((q) => {
|
|
179
|
-
if (!(Number.isFinite(q?.remaining) && typeof q.unit === 'string')) throw new Error(`${provider} answered no quota: ${JSON.stringify(q)}`);
|
|
180
|
-
return q;
|
|
181
|
-
})
|
|
182
|
-
.finally(() => (reading.done = true));
|
|
183
|
-
reading.answer.catch(() => {}); // awaited by the callers; not an unhandled rejection while cached
|
|
184
|
-
this.#readings.set(provider, reading);
|
|
185
|
-
return reading;
|
|
59
|
+
/** Names of the providers it has adapters for */
|
|
60
|
+
get providers() {
|
|
61
|
+
return [...this.#providers];
|
|
186
62
|
}
|
|
187
63
|
|
|
188
|
-
/**
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
return sessionCost(this.#known(provider).settings.price, sessionMsAtMost(sessionOptions(options)));
|
|
64
|
+
/** The accounts with an API key, in the providers' order and then the keys': all, or `provider`'s (throws for an unknown one) */
|
|
65
|
+
accounts(provider) {
|
|
66
|
+
if (provider === undefined) return [...this.#accounts];
|
|
67
|
+
if (!this.#providers.includes(provider)) throw new Error(`unknown provider ${provider} (expected ${this.#providers.join(', ')})`);
|
|
68
|
+
return this.#accounts.filter((a) => a.provider === provider);
|
|
194
69
|
}
|
|
195
70
|
|
|
196
71
|
/**
|
|
197
|
-
* Stores
|
|
198
|
-
*
|
|
199
|
-
*
|
|
200
|
-
* to the programs that have one. Returns { reported: sessions completed, failed: [{ provider, error }] }.
|
|
72
|
+
* Stores the providers' records (duration, proxy traffic) of the ledger's ended sessions that have none yet:
|
|
73
|
+
* { reported, failed: [{ account, error }] }. What cannot be fetched now, and the sessions of accounts without
|
|
74
|
+
* a key here, are left for a later call.
|
|
201
75
|
*/
|
|
202
76
|
async syncUsage() {
|
|
203
|
-
const store = this.#store;
|
|
204
77
|
let reported = 0;
|
|
205
78
|
const failed = [];
|
|
206
|
-
for (const
|
|
79
|
+
for (const account of this.#accounts) {
|
|
207
80
|
try {
|
|
208
|
-
|
|
209
|
-
const usage = await adapter.sessionUsage(settings, apiKey, rows);
|
|
210
|
-
store.tx(() => usage.forEach((u) => store.recordUsage(provider, u)));
|
|
211
|
-
reported += usage.length;
|
|
81
|
+
reported += await account.syncUsage();
|
|
212
82
|
} catch (e) {
|
|
213
|
-
failed.push({
|
|
83
|
+
failed.push({ account: account.id, error: e.message });
|
|
214
84
|
}
|
|
215
85
|
}
|
|
216
86
|
return { reported, failed };
|
|
217
87
|
}
|
|
218
88
|
|
|
219
89
|
/**
|
|
220
|
-
* Sessions opened since `since` (ms), per UTC day,
|
|
221
|
-
* [{ day, provider, client, sessions, minutes, untimed, unsynced, proxyBytes }].
|
|
222
|
-
* record
|
|
223
|
-
*
|
|
224
|
-
* reported them yet); unsynced: sessions without the provider's record.
|
|
90
|
+
* Sessions opened since `since` (ms), per UTC day, account and client:
|
|
91
|
+
* [{ day, provider, account, client, sessions, minutes, untimed, unsynced, proxyBytes }]. minutes: the
|
|
92
|
+
* provider's record (syncUsage), else the open-to-close timing; untimed: sessions with neither; unsynced:
|
|
93
|
+
* those without the record. account: the provider's name alone for sessions whose account is not known.
|
|
225
94
|
*/
|
|
226
95
|
usage({ since }) {
|
|
227
96
|
return this.#store.usage(since);
|
|
228
97
|
}
|
|
229
98
|
|
|
230
|
-
/** Closes every session still open
|
|
99
|
+
/** Closes every session still open and the ledger */
|
|
231
100
|
async close() {
|
|
232
|
-
await Promise.all(
|
|
101
|
+
await Promise.all(this.#accounts.map((a) => a.close()));
|
|
233
102
|
this.#ledger?.close();
|
|
234
103
|
this.#ledger = null;
|
|
235
104
|
}
|
|
236
105
|
|
|
237
|
-
/** close(), for `await using browsers = new OneBrowser(…)` */
|
|
238
106
|
[Symbol.asyncDispose]() {
|
|
239
107
|
return this.close();
|
|
240
108
|
}
|
package/src/ledger.js
CHANGED
|
@@ -6,7 +6,7 @@ import { DatabaseSync } from 'node:sqlite';
|
|
|
6
6
|
/**
|
|
7
7
|
* Where the ledger lives unless a path is given: $ONEBROWSER_DB, else onebrowser/sessions.db in the user's data
|
|
8
8
|
* directory ($XDG_DATA_HOME, default ~/.local/share). One ledger per machine is shared by every project on it, so
|
|
9
|
-
* that the record of
|
|
9
|
+
* that the record of an account's sessions is whole, whichever project opened them.
|
|
10
10
|
*/
|
|
11
11
|
export function defaultLedgerPath() {
|
|
12
12
|
if (process.env.ONEBROWSER_DB) return process.env.ONEBROWSER_DB;
|
|
@@ -14,48 +14,56 @@ export function defaultLedgerPath() {
|
|
|
14
14
|
return path.join(data, 'onebrowser', 'sessions.db');
|
|
15
15
|
}
|
|
16
16
|
|
|
17
|
-
/**
|
|
18
|
-
|
|
19
|
-
* is billed from then on, even if connecting fails), closed when released, and completed with the
|
|
20
|
-
* provider's own record (duration, proxy traffic) once the session has ended and syncUsage has
|
|
21
|
-
* fetched it
|
|
22
|
-
*/
|
|
23
|
-
const SESSIONS = `
|
|
17
|
+
/** Every remote browser session, from when its provider has created it: it is billed from then on, even if connecting fails */
|
|
18
|
+
const SCHEMA = `
|
|
24
19
|
CREATE TABLE sessions (
|
|
25
20
|
provider TEXT NOT NULL,
|
|
21
|
+
account TEXT, -- the account it is billed to (Account.id); empty when not known: recorded
|
|
22
|
+
-- before onebrowser 0.6, by a provider with several accounts here since
|
|
26
23
|
id TEXT NOT NULL, -- the session id its provider gave it
|
|
27
24
|
client TEXT NOT NULL, -- the program that opened it (OneBrowser's client)
|
|
28
25
|
opened_at TEXT NOT NULL, -- when it was asked for; its provider created it right after
|
|
29
26
|
ends_by TEXT NOT NULL, -- when it has ended at the latest, whatever becomes of its process
|
|
30
|
-
closed_at TEXT, -- when
|
|
27
|
+
closed_at TEXT, -- when its provider released it; empty until then, and if it never was
|
|
31
28
|
duration_ms INTEGER, -- the provider's record, once the session has ended
|
|
32
29
|
proxy_bytes INTEGER, -- proxy traffic in the provider's record
|
|
33
30
|
synced_at TEXT, -- when the provider's record was stored; empty = not yet
|
|
34
31
|
PRIMARY KEY (provider, id)
|
|
35
32
|
);
|
|
36
|
-
CREATE INDEX idx_sessions_opened ON sessions(
|
|
33
|
+
CREATE INDEX idx_sessions_opened ON sessions(account, opened_at);
|
|
37
34
|
`;
|
|
38
35
|
|
|
39
36
|
/**
|
|
40
|
-
* The
|
|
41
|
-
*
|
|
42
|
-
*
|
|
37
|
+
* The version of SCHEMA (PRAGMA user_version), and the step to it from each earlier one: a ledger outlives the
|
|
38
|
+
* onebrowser that made it. onebrowser 0.1–0.3 made version 1 and left user_version 0; 0.4 and 0.5 made 2. Each
|
|
39
|
+
* step makes its version as that version was, whatever SCHEMA has become since.
|
|
43
40
|
*/
|
|
44
|
-
const
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
41
|
+
const VERSION = 3;
|
|
42
|
+
const UPGRADES = {
|
|
43
|
+
// 1 → 2: ends_by, as if the sessions of 1 had the default session options (15 min, then 60 s idle)
|
|
44
|
+
1: `ALTER TABLE sessions RENAME TO sessions_1;
|
|
45
|
+
DROP INDEX idx_sessions_opened;
|
|
46
|
+
CREATE TABLE sessions (
|
|
47
|
+
provider TEXT NOT NULL, id TEXT NOT NULL, client TEXT NOT NULL, opened_at TEXT NOT NULL, ends_by TEXT NOT NULL,
|
|
48
|
+
closed_at TEXT, duration_ms INTEGER, proxy_bytes INTEGER, synced_at TEXT, PRIMARY KEY (provider, id)
|
|
49
|
+
);
|
|
50
|
+
CREATE INDEX idx_sessions_opened ON sessions(provider, opened_at);
|
|
51
|
+
INSERT INTO sessions (provider, id, client, opened_at, ends_by, closed_at, duration_ms, proxy_bytes, synced_at)
|
|
52
|
+
SELECT provider, id, client, opened_at, strftime('%Y-%m-%dT%H:%M:%fZ', opened_at, '+16 minutes'), closed_at, duration_ms, proxy_bytes, synced_at
|
|
53
|
+
FROM sessions_1;
|
|
54
|
+
DROP TABLE sessions_1;`,
|
|
55
|
+
// 2 → 3: account, not known for the sessions so far (claim() settles those of a provider with one account)
|
|
56
|
+
2: `ALTER TABLE sessions RENAME TO sessions_2;
|
|
57
|
+
DROP INDEX idx_sessions_opened;
|
|
58
|
+
CREATE TABLE sessions (
|
|
59
|
+
provider TEXT NOT NULL, account TEXT, id TEXT NOT NULL, client TEXT NOT NULL, opened_at TEXT NOT NULL, ends_by TEXT NOT NULL,
|
|
60
|
+
closed_at TEXT, duration_ms INTEGER, proxy_bytes INTEGER, synced_at TEXT, PRIMARY KEY (provider, id)
|
|
61
|
+
);
|
|
62
|
+
CREATE INDEX idx_sessions_opened ON sessions(account, opened_at);
|
|
63
|
+
INSERT INTO sessions (provider, id, client, opened_at, ends_by, closed_at, duration_ms, proxy_bytes, synced_at)
|
|
64
|
+
SELECT provider, id, client, opened_at, ends_by, closed_at, duration_ms, proxy_bytes, synced_at FROM sessions_2;
|
|
65
|
+
DROP TABLE sessions_2;`,
|
|
66
|
+
};
|
|
59
67
|
|
|
60
68
|
/** ms from one ISO time to another */
|
|
61
69
|
const msBetween = (from, to) => `CAST(ROUND((julianday(${to}) - julianday(${from})) * 86400000) AS INTEGER)`;
|
|
@@ -73,29 +81,31 @@ const iso = (ms) => new Date(ms).toISOString();
|
|
|
73
81
|
const int = (x) => (x === null || x === undefined || !Number.isFinite(Number(x)) ? null : Math.trunc(Number(x)));
|
|
74
82
|
|
|
75
83
|
export class Ledger {
|
|
84
|
+
#file;
|
|
76
85
|
#db;
|
|
77
86
|
#stmts = new Map();
|
|
78
87
|
|
|
79
88
|
constructor(file) {
|
|
80
89
|
if (file !== ':memory:') fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
90
|
+
this.#file = file;
|
|
81
91
|
this.#db = new DatabaseSync(file);
|
|
82
92
|
this.#db.exec('PRAGMA journal_mode=WAL; PRAGMA busy_timeout=5000;'); // several processes share it
|
|
83
93
|
this.#migrate();
|
|
84
94
|
}
|
|
85
95
|
|
|
86
|
-
/**
|
|
96
|
+
/** Makes the ledger, or brings an earlier one up to date, in one transaction: another process may be opening it too */
|
|
87
97
|
#migrate() {
|
|
88
98
|
const version = () => this.#db.prepare('PRAGMA user_version').get().user_version;
|
|
89
|
-
if (version() ===
|
|
90
|
-
this
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
}
|
|
99
|
+
if (version() === VERSION) return;
|
|
100
|
+
this.tx(() => {
|
|
101
|
+
if (!this.#db.prepare("SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = 'sessions'").get()) this.#db.exec(SCHEMA);
|
|
102
|
+
else {
|
|
103
|
+
const from = version() || 1;
|
|
104
|
+
if (from > VERSION) throw new Error(`onebrowser: the ledger ${this.#file} was made by a later onebrowser (schema ${from}; this one knows ${VERSION})`);
|
|
105
|
+
for (let v = from; v < VERSION; v++) this.#db.exec(UPGRADES[v]);
|
|
106
|
+
}
|
|
107
|
+
this.#db.exec(`PRAGMA user_version = ${VERSION}`);
|
|
108
|
+
});
|
|
99
109
|
}
|
|
100
110
|
|
|
101
111
|
#q(sql) {
|
|
@@ -104,44 +114,45 @@ export class Ledger {
|
|
|
104
114
|
return s;
|
|
105
115
|
}
|
|
106
116
|
|
|
107
|
-
opened({ provider, id, client, openedAt, endsBy }) {
|
|
108
|
-
this.#q('INSERT INTO sessions (provider, id, client, opened_at, ends_by) VALUES (
|
|
117
|
+
opened({ provider, account, id, client, openedAt, endsBy }) {
|
|
118
|
+
this.#q('INSERT INTO sessions (provider, account, id, client, opened_at, ends_by) VALUES (?,?,?,?,?,?)').run(provider, account, id, client, iso(openedAt), iso(endsBy));
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
closed({ account, id, closedAt }) {
|
|
122
|
+
this.#q('UPDATE sessions SET closed_at=? WHERE account=? AND id=? AND closed_at IS NULL').run(iso(closedAt), account, id);
|
|
109
123
|
}
|
|
110
124
|
|
|
111
|
-
|
|
112
|
-
|
|
125
|
+
/** The provider's sessions whose account is not known are `account`'s */
|
|
126
|
+
claim(provider, account) {
|
|
127
|
+
this.#q('UPDATE sessions SET account=? WHERE provider=? AND account IS NULL').run(account, provider);
|
|
113
128
|
}
|
|
114
129
|
|
|
115
|
-
/** The
|
|
116
|
-
history(
|
|
130
|
+
/** The account's sessions that a quota reading shows, those that ended before `billedBefore` (ms); openSince() has the rest */
|
|
131
|
+
history(account, billedBefore) {
|
|
117
132
|
return {
|
|
118
|
-
/**
|
|
133
|
+
/** Their session time (ms) since `since` (ms), each as long as it can have lasted while it has no timing */
|
|
119
134
|
endedSessionMsSince: (since) =>
|
|
120
|
-
this.#q(`SELECT COALESCE(SUM(${SESSION_MS_AT_MOST}), 0) ms FROM sessions WHERE
|
|
135
|
+
this.#q(`SELECT COALESCE(SUM(${SESSION_MS_AT_MOST}), 0) ms FROM sessions WHERE account=? AND opened_at >= ? AND ${ENDED_AT} < ?`).get(account, iso(since), iso(billedBefore)).ms,
|
|
121
136
|
};
|
|
122
137
|
}
|
|
123
138
|
|
|
124
|
-
/**
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
*/
|
|
128
|
-
openAfter(provider, at) {
|
|
129
|
-
return this.#q(`SELECT ${SESSION_MS_AT_MOST} durationMs FROM sessions WHERE provider=? AND ${ENDED_AT} >= ?`).all(provider, iso(at));
|
|
139
|
+
/** The account's sessions open at any time since `at` (ms): [{ durationMs }], as long as they can last while untimed */
|
|
140
|
+
openSince(account, at) {
|
|
141
|
+
return this.#q(`SELECT ${SESSION_MS_AT_MOST} durationMs FROM sessions WHERE account=? AND ${ENDED_AT} >= ?`).all(account, iso(at));
|
|
130
142
|
}
|
|
131
143
|
|
|
132
|
-
/**
|
|
133
|
-
awaitingUsage(
|
|
134
|
-
return this.#q(
|
|
135
|
-
'SELECT provider, id, opened_at openedAt, closed_at closedAt FROM sessions WHERE synced_at IS NULL AND provider IN (SELECT value FROM json_each(?)) ORDER BY opened_at',
|
|
136
|
-
).all(JSON.stringify(providers));
|
|
144
|
+
/** The account's sessions without their provider's record yet: [{ id, openedAt, closedAt }] (ISO times) */
|
|
145
|
+
awaitingUsage(account) {
|
|
146
|
+
return this.#q('SELECT id, opened_at openedAt, closed_at closedAt FROM sessions WHERE synced_at IS NULL AND account=? ORDER BY opened_at').all(account);
|
|
137
147
|
}
|
|
138
148
|
|
|
139
|
-
recordUsage(
|
|
140
|
-
this.#q('UPDATE sessions SET duration_ms=?, proxy_bytes=?, synced_at=? WHERE
|
|
149
|
+
recordUsage(account, { id, durationMs, proxyBytes }) {
|
|
150
|
+
this.#q('UPDATE sessions SET duration_ms=?, proxy_bytes=?, synced_at=? WHERE account=? AND id=?').run(int(durationMs), int(proxyBytes), iso(Date.now()), account, id);
|
|
141
151
|
}
|
|
142
152
|
|
|
153
|
+
/** Runs fn in a transaction that takes the write lock at once: another process may be writing too */
|
|
143
154
|
tx(fn) {
|
|
144
|
-
this.#db.exec('BEGIN');
|
|
155
|
+
this.#db.exec('BEGIN IMMEDIATE');
|
|
145
156
|
try {
|
|
146
157
|
const r = fn();
|
|
147
158
|
this.#db.exec('COMMIT');
|
|
@@ -155,10 +166,10 @@ export class Ledger {
|
|
|
155
166
|
/** See OneBrowser.usage() */
|
|
156
167
|
usage(since) {
|
|
157
168
|
return this.#q(
|
|
158
|
-
`SELECT substr(opened_at, 1, 10) day, provider, client, COUNT(*) sessions,
|
|
169
|
+
`SELECT substr(opened_at, 1, 10) day, provider, COALESCE(account, provider) account, client, COUNT(*) sessions,
|
|
159
170
|
COALESCE(SUM(${SESSION_MS}), 0) / 60000.0 minutes, SUM(${SESSION_MS} IS NULL) untimed,
|
|
160
171
|
SUM(synced_at IS NULL) unsynced, COALESCE(SUM(proxy_bytes), 0) proxyBytes
|
|
161
|
-
FROM sessions WHERE opened_at >= ? GROUP BY day, provider, client ORDER BY day, provider, client`,
|
|
172
|
+
FROM sessions WHERE opened_at >= ? GROUP BY day, provider, account, client ORDER BY day, provider, account, client`,
|
|
162
173
|
).all(iso(since));
|
|
163
174
|
}
|
|
164
175
|
|
package/src/options.js
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { KEEP_ALIVE_MS } from './session.js';
|
|
2
|
+
|
|
3
|
+
/** Session options when openSession() is not given them */
|
|
4
|
+
const SESSION_DEFAULTS = {
|
|
5
|
+
proxyCountry: null,
|
|
6
|
+
idleTimeoutSec: 60,
|
|
7
|
+
maxDurationMs: 15 * 60_000,
|
|
8
|
+
};
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* The session options of openSession() with the defaults filled in; throws when one is unknown or out of range,
|
|
12
|
+
* so that a program can check its configuration before it opens any session:
|
|
13
|
+
* proxyCountry country code: the provider's proxy there; null: its default egress
|
|
14
|
+
* idleTimeoutSec a session whose client is gone ends this long after at the latest, or after the provider's
|
|
15
|
+
* minIdleTimeoutSec if longer (≥ 2 × KEEP_ALIVE_MS)
|
|
16
|
+
* maxDurationMs a session ends this long after it was opened at the latest
|
|
17
|
+
*/
|
|
18
|
+
export function sessionOptions(options = {}) {
|
|
19
|
+
const unknown = Object.keys(options).filter((k) => !Object.hasOwn(SESSION_DEFAULTS, k));
|
|
20
|
+
if (unknown.length) throw new Error(`unknown session option ${unknown.join(', ')} (expected ${Object.keys(SESSION_DEFAULTS).join(', ')})`);
|
|
21
|
+
const o = { ...SESSION_DEFAULTS, ...options };
|
|
22
|
+
if (!(Number.isFinite(o.idleTimeoutSec) && o.idleTimeoutSec * 1000 >= 2 * KEEP_ALIVE_MS)) {
|
|
23
|
+
throw new Error(`idleTimeoutSec must be at least ${(2 * KEEP_ALIVE_MS) / 1000} (twice the keep-alive interval), not ${o.idleTimeoutSec}`);
|
|
24
|
+
}
|
|
25
|
+
if (!(Number.isFinite(o.maxDurationMs) && o.maxDurationMs > 0)) throw new Error(`maxDurationMs must be a number > 0, not ${o.maxDurationMs}`);
|
|
26
|
+
return o;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** How long a session lasts at most, whatever becomes of its process: closed at maxDurationMs, else ended by its provider once idle */
|
|
30
|
+
export const sessionMsAtMost = (o, settings) => o.maxDurationMs + Math.max(o.idleTimeoutSec, settings.minIdleTimeoutSec) * 1000;
|
package/src/providers/anchor.js
CHANGED
|
@@ -16,6 +16,7 @@ export const defaults = {
|
|
|
16
16
|
// (credits_used) bill more, about $0.06 an hour and a step of $0.003 in the second minute, which this covers
|
|
17
17
|
price: { perSession: 0.01, perHour: 0.15, stepSec: 60 },
|
|
18
18
|
billingDelayMs: 5 * 60_000, // seen within 10 s
|
|
19
|
+
minIdleTimeoutSec: 60, // idle_timeout is in whole minutes, at least 1
|
|
19
20
|
};
|
|
20
21
|
|
|
21
22
|
const headers = (apiKey) => ({ 'anchor-api-key': apiKey });
|
package/src/providers/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { fullPrice } from '../cost.js';
|
|
2
2
|
import * as anchor from './anchor.js';
|
|
3
3
|
import * as firecrawl from './firecrawl.js';
|
|
4
4
|
import * as hyperbrowser from './hyperbrowser.js';
|
|
@@ -8,28 +8,22 @@ import * as steel from './steel.js';
|
|
|
8
8
|
import * as tinyfish from './tinyfish.js';
|
|
9
9
|
|
|
10
10
|
/**
|
|
11
|
-
* Provider adapters
|
|
12
|
-
* defaults
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
* createSession(settings, apiKey,
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
* session because the account's quota is used up
|
|
21
|
-
* releaseSession(settings, apiKey, session) ends the session; throws on failure
|
|
11
|
+
* Provider adapters by name: the only provider-specific code. Each exports:
|
|
12
|
+
* defaults its settings, which OneBrowser's `providers` option overrides:
|
|
13
|
+
* price what its sessions cost, in the unit of its quota (cost.js)
|
|
14
|
+
* billingDelayMs how long after a session has ended its quota reading shows it, at the latest
|
|
15
|
+
* minIdleTimeoutSec the shortest idle timeout it takes (default 0)
|
|
16
|
+
* and its own API base URLs, plan facts such as monthlyHours
|
|
17
|
+
* createSession(settings, apiKey, { proxyCountry, idleTimeoutSec, maxDurationMs }) → { id, cdpUrl, egress }
|
|
18
|
+
* throws QuotaError (errors.js) when the provider refuses the session for the account's quota
|
|
19
|
+
* releaseSession(settings, apiKey, session)
|
|
22
20
|
* sessionUsage(settings, apiKey, [{ id, openedAt, closedAt }]) → [{ id, durationMs, proxyBytes }]
|
|
23
|
-
* the record of those
|
|
24
|
-
* API key cannot read, the ledger's timing (closedAt is empty for a session no process closed); throws on
|
|
25
|
-
* failure
|
|
21
|
+
* the provider's record of those that have ended, or the ledger's timing where its API key cannot read one
|
|
26
22
|
* quota(settings, apiKey, history) → { remaining, unit, renewsAt? }
|
|
27
|
-
* what
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
* Their requests go through http.js, which turns every status other than 2xx into an HttpError: an adapter
|
|
32
|
-
* handles only the statuses that mean something of their own.
|
|
23
|
+
* what the account has left in the provider's unit, and when it renews by itself (ms); history: the
|
|
24
|
+
* ledger's sessions that the reading shows (Ledger.history), for a provider that cannot count its usage
|
|
25
|
+
* Every function throws when it fails. Requests go through http.js, which throws HttpError for a status other
|
|
26
|
+
* than 2xx.
|
|
33
27
|
*/
|
|
34
28
|
export const ADAPTERS = { tinyfish, steel, hyperbrowser, lightpanda, kernel, anchor, firecrawl };
|
|
35
29
|
|
|
@@ -41,12 +35,15 @@ export function checkAdapter(name, adapter) {
|
|
|
41
35
|
if (missing.length) throw new Error(`provider ${name}: its adapter lacks ${missing.join(', ')}`);
|
|
42
36
|
}
|
|
43
37
|
|
|
44
|
-
/**
|
|
45
|
-
export function
|
|
38
|
+
/** An adapter's settings with the defaults of those every adapter has; throws when one of them is missing or wrong */
|
|
39
|
+
export function fullSettings(name, { price, billingDelayMs, minIdleTimeoutSec = 0, ...own }) {
|
|
40
|
+
const atLeast0 = (x) => Number.isFinite(x) && x >= 0;
|
|
46
41
|
try {
|
|
47
|
-
|
|
42
|
+
const full = fullPrice(price);
|
|
43
|
+
if (!atLeast0(billingDelayMs)) throw new Error(`billingDelayMs must be a number ≥ 0, not ${billingDelayMs}`);
|
|
44
|
+
if (!atLeast0(minIdleTimeoutSec)) throw new Error(`minIdleTimeoutSec must be a number ≥ 0, not ${minIdleTimeoutSec}`);
|
|
45
|
+
return { ...own, price: full, billingDelayMs, minIdleTimeoutSec };
|
|
48
46
|
} catch (e) {
|
|
49
47
|
throw new Error(`provider ${name}: ${e.message}`);
|
|
50
48
|
}
|
|
51
|
-
if (!(Number.isFinite(settings.billingDelayMs) && settings.billingDelayMs >= 0)) throw new Error(`provider ${name}: billingDelayMs must be a number ≥ 0, not ${settings.billingDelayMs}`);
|
|
52
49
|
}
|