@askalf/dario 6.18.4 → 6.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/account-sync.d.ts +174 -0
- package/dist/account-sync.js +351 -0
- package/dist/accounts.d.ts +66 -1
- package/dist/accounts.js +200 -13
- package/dist/cc-template-data.json +3 -3
- package/dist/cli.js +51 -2
- package/dist/live-fingerprint.d.ts +1 -1
- package/dist/live-fingerprint.js +1 -1
- package/dist/proxy.d.ts +8 -0
- package/dist/proxy.js +121 -8
- package/docs/admin-api.md +8 -0
- package/docs/configuration.md +1 -0
- package/docs/docker.md +35 -4
- package/docs/multi-instance.md +37 -3
- package/package.json +1 -1
package/dist/proxy.js
CHANGED
|
@@ -20,6 +20,7 @@ import { describeTemplate, detectDrift, checkCCCompat, probeInstalledCCVersion }
|
|
|
20
20
|
import { AccountPool, computeStickyKey, parseRateLimits, modelFamily, isInAuthCooldown, authCooldownMs, accountIneligibility, reportedAccountStatus, activeParkedBuckets, reconcilePoolAccounts, resolvePoolStrategy, resolvePoolHeadroomFloor, parsePoolHeadroomFloor, describePoolStrategy, utilFreshness, rateLimitWindow, accountAction, accountPeers, distinctAccounts, describeRejection, maskEmail } from './pool.js';
|
|
21
21
|
import { backfillIdentity } from './accounts.js';
|
|
22
22
|
import { PoolSync, DEFAULT_POOL_SYNC_INTERVAL_MS } from './pool-sync.js';
|
|
23
|
+
import { AccountSync, DEFAULT_ACCOUNT_SYNC_INTERVAL_MS } from './account-sync.js';
|
|
23
24
|
import { Analytics, billingBucketFromClaim, costOfTokensFailClosed, formatUsageLogLine, SUBSCRIPTION_CLAIMS, consumerFromHeader, consumerFromBody, CONSUMER_HEADER, CODEX_CLAIM, CREDITS_CLAIM } from './analytics.js';
|
|
24
25
|
import { Ledger, resolveLedgerPath, ledgerDisabledByEnv } from './ledger.js';
|
|
25
26
|
import { renderPrometheus } from './metrics.js';
|
|
@@ -29,7 +30,7 @@ import { OverageGuard, buildHaltErrorBody } from './overage-guard.js';
|
|
|
29
30
|
import { notify as osNotify } from './notify.js';
|
|
30
31
|
import { grantAge, grantThresholds, worstGrantLevel, describeGrantAge } from './refresh-grant.js';
|
|
31
32
|
import { resolveSeatPin, SEAT_PIN_HEADER, SEAT_PIN_TOKEN_HEADER } from './seat-pin.js';
|
|
32
|
-
import { loadAllAccounts, loadAccount, saveAccount, refreshAccountToken, resyncLoginFromCredentialsIfStale, ensureLoginCredentialsInPool, mirrorLoginToCredentials, withObservedOrganization } from './accounts.js';
|
|
33
|
+
import { loadAllAccounts, loadAccount, saveAccount, refreshAccountToken, adoptSharedCredentials, credentialsAreNewer, grantedAfter, localAccountRemovals, resyncLoginFromCredentialsIfStale, ensureLoginCredentialsInPool, MIGRATED_LOGIN_ALIAS, mirrorLoginToCredentials, withObservedOrganization } from './accounts.js';
|
|
33
34
|
import { handleAdminRequest } from './admin-api.js';
|
|
34
35
|
import { createTokenBucket } from './rate-limit.js';
|
|
35
36
|
import { getOpenAIBackend, isOpenAIModel, forwardToOpenAI, isMiniMaxBackend, MINIMAX_MODELS } from './openai-backend.js';
|
|
@@ -1654,6 +1655,27 @@ export async function startProxy(opts = {}) {
|
|
|
1654
1655
|
// with zero accounts and returns a clean 503 until one is added via
|
|
1655
1656
|
// POST /admin/login/*, taking effect with no restart (see onAccountsChanged).
|
|
1656
1657
|
const adminEnabled = process.env.DARIO_ADMIN === '1';
|
|
1658
|
+
// Shared account registry (account-sync.ts): opt-in, rides the refresh-lock
|
|
1659
|
+
// service, fails open. Merged before the pool loads, so a replica that
|
|
1660
|
+
// boots with an empty accounts directory starts with its peers' accounts.
|
|
1661
|
+
// Claude seats only, so not under --no-claude-auth.
|
|
1662
|
+
const lockUrl = process.env['DARIO_REFRESH_LOCK_URL'];
|
|
1663
|
+
const accountSync = opts.poolSharedAccounts && lockUrl && !opts.noClaudeAuth
|
|
1664
|
+
? new AccountSync({
|
|
1665
|
+
baseUrl: lockUrl,
|
|
1666
|
+
token: process.env['DARIO_REFRESH_LOCK_TOKEN'] ?? '',
|
|
1667
|
+
intervalMs: DEFAULT_ACCOUNT_SYNC_INTERVAL_MS,
|
|
1668
|
+
log: (line) => console.error(line),
|
|
1669
|
+
})
|
|
1670
|
+
: null;
|
|
1671
|
+
if (opts.poolSharedAccounts && !lockUrl) {
|
|
1672
|
+
console.error('[dario] --pool-shared-accounts needs DARIO_REFRESH_LOCK_URL (the lock service holds the registry); serving the accounts on this instance\'s disk only');
|
|
1673
|
+
}
|
|
1674
|
+
else if (opts.poolSharedAccounts && opts.noClaudeAuth) {
|
|
1675
|
+
console.error('[dario] --pool-shared-accounts is off under --no-claude-auth: the registry holds Claude seats, and this instance loads none');
|
|
1676
|
+
}
|
|
1677
|
+
if (accountSync)
|
|
1678
|
+
await accountSync.syncOnce();
|
|
1657
1679
|
const accountsList = await loadAllAccounts();
|
|
1658
1680
|
const poolStrategy = resolvePoolStrategy(opts.poolStrategy);
|
|
1659
1681
|
const poolHeadroomFloor = resolvePoolHeadroomFloor(opts.poolHeadroomFloor);
|
|
@@ -1681,9 +1703,39 @@ export async function startProxy(opts = {}) {
|
|
|
1681
1703
|
}
|
|
1682
1704
|
}
|
|
1683
1705
|
};
|
|
1706
|
+
// Take a peer instance's newer credentials for this seat from the refresh
|
|
1707
|
+
// lock service before deciding whether to refresh it (dario#1733). Without
|
|
1708
|
+
// this, a replica kept serving and refreshing with the pair it loaded, and
|
|
1709
|
+
// its refresh token had been rotated away by the peer's last refresh. A
|
|
1710
|
+
// no-op without DARIO_REFRESH_LOCK_URL, and never fatal: the refresh
|
|
1711
|
+
// decision below still runs on what the seat already has.
|
|
1712
|
+
const adoptPeerCredentials = async (alias) => {
|
|
1713
|
+
const seat = pool.get(alias);
|
|
1714
|
+
if (!seat)
|
|
1715
|
+
return;
|
|
1716
|
+
const shared = await adoptSharedCredentials(seat).catch(() => null);
|
|
1717
|
+
if (!shared)
|
|
1718
|
+
return;
|
|
1719
|
+
// The seat can move on while the lock service answers (a login, a
|
|
1720
|
+
// refresh, a removal); replace it only if it is still there and older.
|
|
1721
|
+
const live = pool.get(alias);
|
|
1722
|
+
if (!live || !credentialsAreNewer(shared, live))
|
|
1723
|
+
return;
|
|
1724
|
+
// add(), not updateTokens(): a newer grant carries its own grantedAt, and
|
|
1725
|
+
// add() is what starts a re-granted seat's live state fresh.
|
|
1726
|
+
pool.add(alias, {
|
|
1727
|
+
accessToken: shared.accessToken, refreshToken: shared.refreshToken, expiresAt: shared.expiresAt,
|
|
1728
|
+
deviceId: shared.deviceId, accountUuid: shared.accountUuid, grantedAt: shared.grantedAt,
|
|
1729
|
+
organizationId: shared.organizationId, accountId: shared.accountId, accountEmail: shared.accountEmail,
|
|
1730
|
+
rateLimitTier: shared.rateLimitTier, seatTier: shared.seatTier,
|
|
1731
|
+
});
|
|
1732
|
+
await mirrorLoginToCredentials(shared).catch((err) => {
|
|
1733
|
+
console.error(`[dario] login→credentials.json mirror failed (peer credentials) for ${alias}: ${err instanceof Error ? err.message : err}`);
|
|
1734
|
+
});
|
|
1735
|
+
console.error(`[dario] Adopted credentials another instance refreshed for account ${alias}.`);
|
|
1736
|
+
};
|
|
1684
1737
|
// Shared pool state across instances (docs/multi-instance.md): opt-in,
|
|
1685
1738
|
// rides the refresh-lock service, fails open. Off → byte-identical to before.
|
|
1686
|
-
const lockUrl = process.env['DARIO_REFRESH_LOCK_URL'];
|
|
1687
1739
|
const poolSync = opts.poolSharedState && lockUrl
|
|
1688
1740
|
? new PoolSync(pool, {
|
|
1689
1741
|
baseUrl: lockUrl,
|
|
@@ -1819,7 +1871,9 @@ export async function startProxy(opts = {}) {
|
|
|
1819
1871
|
// token (invalid_grant) just logs and leaves the account expired; the auth
|
|
1820
1872
|
// gate below then surfaces it. Skipped in --no-claude-auth mode.
|
|
1821
1873
|
if (!opts.noClaudeAuth) {
|
|
1822
|
-
await Promise.all(pool.all().map(async (
|
|
1874
|
+
await Promise.all(pool.all().map(async (seat) => {
|
|
1875
|
+
await adoptPeerCredentials(seat.alias);
|
|
1876
|
+
const acc = pool.get(seat.alias) ?? seat;
|
|
1823
1877
|
if (acc.expiresAt >= Date.now() + 45 * 60 * 1000)
|
|
1824
1878
|
return;
|
|
1825
1879
|
try {
|
|
@@ -1890,7 +1944,9 @@ export async function startProxy(opts = {}) {
|
|
|
1890
1944
|
if (opts.noClaudeAuth)
|
|
1891
1945
|
return; // never touch the Claude token in OpenAI-only mode
|
|
1892
1946
|
watchGrants();
|
|
1893
|
-
for (const
|
|
1947
|
+
for (const seat of pool.all()) {
|
|
1948
|
+
await adoptPeerCredentials(seat.alias);
|
|
1949
|
+
const acc = pool.get(seat.alias) ?? seat;
|
|
1894
1950
|
if (acc.expiresAt < Date.now() + 45 * 60 * 1000) {
|
|
1895
1951
|
try {
|
|
1896
1952
|
const saved = await loadAccount(acc.alias);
|
|
@@ -1948,7 +2004,15 @@ export async function startProxy(opts = {}) {
|
|
|
1948
2004
|
console.error('[dario] Not authenticated. Run `dario login` first.');
|
|
1949
2005
|
process.exit(1);
|
|
1950
2006
|
}
|
|
1951
|
-
|
|
2007
|
+
// The startup sync deletes the login seat when the shared account
|
|
2008
|
+
// registry removed it. Restoring it here from credentials.json would
|
|
2009
|
+
// serve a removed account until the next sync deleted it again, on every
|
|
2010
|
+
// restart, so a login granted before the removal stays out.
|
|
2011
|
+
const loginRemovedAt = accountSync?.removalOf(MIGRATED_LOGIN_ALIAS);
|
|
2012
|
+
const recovered = await ensureLoginCredentialsInPool(MIGRATED_LOGIN_ALIAS, { removedAt: loginRemovedAt });
|
|
2013
|
+
if (!recovered && loginRemovedAt !== undefined) {
|
|
2014
|
+
console.error('[dario] shared accounts: not restoring the `login` account from credentials.json: it was removed from the shared account registry after that login. This instance starts with no accounts and takes them from the registry as they are added.');
|
|
2015
|
+
}
|
|
1952
2016
|
if (recovered) {
|
|
1953
2017
|
const acc = await loadAccount(recovered);
|
|
1954
2018
|
if (acc) {
|
|
@@ -1970,9 +2034,9 @@ export async function startProxy(opts = {}) {
|
|
|
1970
2034
|
}
|
|
1971
2035
|
}
|
|
1972
2036
|
if (pool.size === 0) {
|
|
1973
|
-
// Reached only under admin bootstrap
|
|
1974
|
-
//
|
|
1975
|
-
// Math.min([]) === Infinity.
|
|
2037
|
+
// Reached only under admin bootstrap, api-key mode, or a login the shared
|
|
2038
|
+
// account registry removed (the not-authenticated case exited above).
|
|
2039
|
+
// Report the empty pool plainly instead of Math.min([]) === Infinity.
|
|
1976
2040
|
if (adminEnabled) {
|
|
1977
2041
|
console.warn('[dario] Starting with no accounts (admin mode). Add one via POST /admin/login/start; LLM requests return 503 until then.');
|
|
1978
2042
|
}
|
|
@@ -2399,6 +2463,48 @@ export async function startProxy(opts = {}) {
|
|
|
2399
2463
|
else {
|
|
2400
2464
|
prewarmModelCatalog(catalogDeps);
|
|
2401
2465
|
}
|
|
2466
|
+
if (accountSync) {
|
|
2467
|
+
// Every sync after the startup one updates the live pool as well as the
|
|
2468
|
+
// accounts directory. Seat by seat, never reconcilePoolAccounts: a whole
|
|
2469
|
+
// reload from disk would also put back a file's older tokens over a seat
|
|
2470
|
+
// that refreshed since the sync read it.
|
|
2471
|
+
accountSync.attach((alias) => pool.get(alias), async ({ records, removed }) => {
|
|
2472
|
+
const before = pool.size;
|
|
2473
|
+
for (const rec of records) {
|
|
2474
|
+
// This loop awaits between seats, so an admin removal can land
|
|
2475
|
+
// after the sync checked its records; it wins over a record
|
|
2476
|
+
// granted before it.
|
|
2477
|
+
if (!grantedAfter(rec, localAccountRemovals().get(rec.alias)))
|
|
2478
|
+
continue;
|
|
2479
|
+
const live = pool.get(rec.alias);
|
|
2480
|
+
if (live && !credentialsAreNewer(rec, live))
|
|
2481
|
+
continue;
|
|
2482
|
+
pool.add(rec.alias, {
|
|
2483
|
+
accessToken: rec.accessToken, refreshToken: rec.refreshToken, expiresAt: rec.expiresAt,
|
|
2484
|
+
deviceId: rec.deviceId, accountUuid: rec.accountUuid, grantedAt: rec.grantedAt,
|
|
2485
|
+
organizationId: rec.organizationId, accountId: rec.accountId, accountEmail: rec.accountEmail,
|
|
2486
|
+
rateLimitTier: rec.rateLimitTier, seatTier: rec.seatTier,
|
|
2487
|
+
});
|
|
2488
|
+
await mirrorLoginToCredentials(rec).catch((err) => {
|
|
2489
|
+
console.error(`[dario] login→credentials.json mirror failed (shared accounts) for ${rec.alias}: ${err instanceof Error ? err.message : err}`);
|
|
2490
|
+
});
|
|
2491
|
+
// The registry lags a peer's refresh by up to one sync; the lock
|
|
2492
|
+
// service holds the rotation itself.
|
|
2493
|
+
await adoptPeerCredentials(rec.alias);
|
|
2494
|
+
}
|
|
2495
|
+
for (const { alias, removedAt } of removed) {
|
|
2496
|
+
const live = pool.get(alias);
|
|
2497
|
+
// A seat granted after the removal is a later login and stays.
|
|
2498
|
+
if (live && !grantedAfter(live, removedAt))
|
|
2499
|
+
pool.remove(alias);
|
|
2500
|
+
}
|
|
2501
|
+
announceSameAccounts();
|
|
2502
|
+
if (before === 0 && pool.size > 0)
|
|
2503
|
+
retryModelCatalogNow(catalogDeps);
|
|
2504
|
+
});
|
|
2505
|
+
console.log(` Pool shared accounts: on (via ${lockUrl}, syncing every ${accountSync.intervalMs / 1000}s; fails open)`);
|
|
2506
|
+
accountSync.start();
|
|
2507
|
+
}
|
|
2402
2508
|
const ERR_UNAUTH = JSON.stringify({ error: 'Unauthorized', message: 'Invalid or missing API key' });
|
|
2403
2509
|
const ERR_FORBIDDEN = JSON.stringify({ error: 'Forbidden', message: 'Path not allowed. Supported paths: POST /v1/messages, POST /v1/messages/count_tokens, POST /v1/chat/completions, GET /v1/models' });
|
|
2404
2510
|
const ERR_METHOD = JSON.stringify({ error: 'Method not allowed' });
|
|
@@ -2907,6 +3013,10 @@ export async function startProxy(opts = {}) {
|
|
|
2907
3013
|
// waiting out the retry window (#636).
|
|
2908
3014
|
if (size > 0)
|
|
2909
3015
|
retryModelCatalogNow(catalogDeps);
|
|
3016
|
+
// Publish the change now rather than at the next interval; each
|
|
3017
|
+
// peer applies it on its own next sync.
|
|
3018
|
+
if (accountSync)
|
|
3019
|
+
void accountSync.syncOnce();
|
|
2910
3020
|
}
|
|
2911
3021
|
catch (err) {
|
|
2912
3022
|
console.error(`[dario] admin: pool hot-reload failed: ${err instanceof Error ? err.message : err}`);
|
|
@@ -3129,6 +3239,8 @@ export async function startProxy(opts = {}) {
|
|
|
3129
3239
|
distinctAccounts: distinctAccounts(pool.all()),
|
|
3130
3240
|
// Shared pool state (pool-sync.ts) — null when off.
|
|
3131
3241
|
sharedState: poolSync ? poolSync.status() : null,
|
|
3242
|
+
// Shared account registry (account-sync.ts), null when off.
|
|
3243
|
+
sharedAccounts: accountSync ? accountSync.status() : null,
|
|
3132
3244
|
accounts,
|
|
3133
3245
|
}));
|
|
3134
3246
|
return;
|
|
@@ -6763,6 +6875,7 @@ export async function startProxy(opts = {}) {
|
|
|
6763
6875
|
clearInterval(presenceInterval);
|
|
6764
6876
|
clearInterval(refreshInterval);
|
|
6765
6877
|
poolSync?.stop();
|
|
6878
|
+
accountSync?.stop();
|
|
6766
6879
|
// Stop accepting connections now; responses already streaming keep going.
|
|
6767
6880
|
// Tokens and what the ledger already holds are written while the drain
|
|
6768
6881
|
// waits. The ledger, the key store and the request log close only after
|
package/docs/admin-api.md
CHANGED
|
@@ -122,6 +122,14 @@ pool view; a headless operator needs only the admin token to watch headroom.
|
|
|
122
122
|
Completing a login for an alias that is already in the pool re-grants it: the
|
|
123
123
|
seat starts fresh — no carried-over rejection, cool-down or identity.
|
|
124
124
|
|
|
125
|
+
With `--pool-shared-accounts` on (see
|
|
126
|
+
[Sharing the account list](./multi-instance.md#sharing-the-account-list)), a
|
|
127
|
+
login or removal made here is published to the shared registry at once, and
|
|
128
|
+
every other instance takes it on its next sync, within 15 seconds. Provisioning
|
|
129
|
+
one instance this way provisions them all, and a new replica starts with the
|
|
130
|
+
accounts already in the registry. `GET /accounts` then carries a
|
|
131
|
+
`sharedAccounts` block with the sync's state.
|
|
132
|
+
|
|
125
133
|
## Pinning a request to one seat
|
|
126
134
|
|
|
127
135
|
`x-dario-account: <alias>` on `POST /v1/messages` (or `/v1/chat/completions`) routes that one request to the named pool account with **no failover**: no headroom selection, no sticky rebinding, no peer retry on 401/429, no Codex leg. The upstream status comes back as-is. It is the primitive behind `dario accounts check <alias>` — the answer to "does this seat serve Sonnet right now?" without copying a credential anywhere or restarting anything.
|
package/docs/configuration.md
CHANGED
|
@@ -67,6 +67,7 @@ Halts the proxy when an upstream response reports `representative-claim: overage
|
|
|
67
67
|
| `DARIO_REFRESH_LOCK_TOKEN` | — | unset | bearer for the lock service |
|
|
68
68
|
| `DARIO_POOL_SHARED_STATE` | `--pool-shared-state` | off | share rate-limit readings and sticky bindings with the other instances through the lock service; fails open |
|
|
69
69
|
| `DARIO_POOL_SHARED_STATE_INTERVAL_MS` | `--pool-shared-state-interval=MS` | `2000` | how often to pull peers' readings |
|
|
70
|
+
| `DARIO_POOL_SHARED_ACCOUNTS` | `--pool-shared-accounts` | off | keep one list of Claude accounts across the instances through the lock service, merged at startup and every 15s, so a new replica starts with its peers' accounts and adds and removals reach every instance; the registry holds every seat's credentials; fails open ([details](./multi-instance.md#sharing-the-account-list)) |
|
|
70
71
|
|
|
71
72
|
## Template fidelity
|
|
72
73
|
|
package/docs/docker.md
CHANGED
|
@@ -169,6 +169,21 @@ readinessProbe:
|
|
|
169
169
|
> [admin API](./admin-api.md). Provision before relying on readiness, or use a
|
|
170
170
|
> long-window `startupProbe` around the bootstrap.
|
|
171
171
|
|
|
172
|
+
A Service routes only to ready pods, so the admin API of a pod that is not
|
|
173
|
+
ready yet cannot be reached through it. Two ways in that leave the probes as
|
|
174
|
+
they are:
|
|
175
|
+
|
|
176
|
+
- `kubectl port-forward pod/<pod> 3456:3456` connects to the pod itself,
|
|
177
|
+
ready or not, and the admin calls above work against `localhost:3456`.
|
|
178
|
+
- A second Service for the admin calls with `publishNotReadyAddresses: true`
|
|
179
|
+
keeps routing to pods that are not ready. Keep it cluster-internal: it
|
|
180
|
+
reaches a pod before it has an account, which is the point.
|
|
181
|
+
|
|
182
|
+
With more than one replica, `--pool-shared-accounts` (below) removes the
|
|
183
|
+
bootstrap from every pod after the first: a pod that starts with an empty
|
|
184
|
+
accounts directory loads its peers' accounts before the pool starts, and
|
|
185
|
+
reports ready as soon as they do.
|
|
186
|
+
|
|
172
187
|
## Kubernetes example
|
|
173
188
|
|
|
174
189
|
```yaml
|
|
@@ -202,7 +217,7 @@ spec:
|
|
|
202
217
|
volumeMounts:
|
|
203
218
|
- { name: config, mountPath: /home/dario/.dario }
|
|
204
219
|
livenessProbe:
|
|
205
|
-
httpGet: { path: /
|
|
220
|
+
httpGet: { path: /livez, port: 3456 }
|
|
206
221
|
periodSeconds: 30
|
|
207
222
|
readinessProbe:
|
|
208
223
|
httpGet: { path: /health, port: 3456 }
|
|
@@ -224,9 +239,25 @@ spec:
|
|
|
224
239
|
ports: [{ port: 3456, targetPort: 3456 }]
|
|
225
240
|
```
|
|
226
241
|
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
242
|
+
Keep `replicas: 1` unless the replicas share a refresh lock: OAuth refresh
|
|
243
|
+
tokens are single use, and two pods holding the same account without one
|
|
244
|
+
race to refresh it. For more than one replica, run the
|
|
245
|
+
[refresh lock](./multi-instance.md#turning-the-refresh-lock-on) and, so a new
|
|
246
|
+
pod starts with the others' accounts and an account added or removed on one
|
|
247
|
+
pod reaches every pod,
|
|
248
|
+
[`--pool-shared-accounts`](./multi-instance.md#sharing-the-account-list):
|
|
249
|
+
|
|
250
|
+
```yaml
|
|
251
|
+
env:
|
|
252
|
+
- { name: DARIO_REFRESH_LOCK_URL, value: "http://dario-refresh-lock:8080" }
|
|
253
|
+
- { name: DARIO_REFRESH_LOCK_TOKEN, valueFrom: { secretKeyRef: { name: dario-lock, key: token } } }
|
|
254
|
+
- { name: DARIO_POOL_SHARED_ACCOUNTS, value: "1" }
|
|
255
|
+
- { name: DARIO_POOL_SHARED_STATE, value: "1" }
|
|
256
|
+
```
|
|
257
|
+
|
|
258
|
+
The registry holds every seat's credentials, so the lock service and its Redis
|
|
259
|
+
need the same care as the accounts volume; see
|
|
260
|
+
[Sharing the account list](./multi-instance.md#sharing-the-account-list).
|
|
230
261
|
|
|
231
262
|
## Image updates
|
|
232
263
|
|
package/docs/multi-instance.md
CHANGED
|
@@ -20,6 +20,8 @@ Two instances holding the same account both notice the token is near expiry, and
|
|
|
20
20
|
|
|
21
21
|
A plain mutex does not fix this. The loser waits, acquires the lock, and then refreshes with a token that is *already* stale — it just loses more slowly. The fix is that the loser **adopts the winner's fresh credentials** instead of attempting its own. That is what dario's refresh lock does.
|
|
22
22
|
|
|
23
|
+
The handoff has to outlast the gap between replicas, not just the race. Each instance checks its tokens every 15 minutes on its own phase, so the second one can come back well after the first refreshed. The lock service keeps the latest credentials for 30 days, and every instance reads them at startup and on each tick before deciding to refresh. Until [#1733](https://github.com/askalf/dario/issues/1733) that window was five minutes, and a replica that came back later refreshed with a dead token.
|
|
24
|
+
|
|
23
25
|
### 2. Rate-limit accounting — **solved with shared state**
|
|
24
26
|
|
|
25
27
|
`pool.ts` keeps `accounts: Map<string, PoolAccount>` in process memory, and updates it only from rate-limit headers on responses *that instance* saw.
|
|
@@ -88,9 +90,13 @@ Not suitable for airgapped deployments, and it adds an internet dependency to a
|
|
|
88
90
|
|
|
89
91
|
### It fails open, deliberately
|
|
90
92
|
|
|
91
|
-
|
|
93
|
+
On any lock-service error (bad response, network failure, timeout), dario first waits if it can. While the access token still has more than 20 minutes left, the refresh is put off to a later tick, because refreshing alone is what can strand a peer. Inside 20 minutes dario refreshes on its own, exactly as if no lock were configured.
|
|
94
|
+
|
|
95
|
+
The lock is resilience layered on top of dario's job, not a new dependency dario's core function needs. An outage of your Redis must not stop your proxy serving traffic. The cost of that choice is that an outage lasting into a token's last 20 minutes puts you back to the plain race, so do not treat the lock as a hard guarantee.
|
|
92
96
|
|
|
93
|
-
|
|
97
|
+
### Upgrading the lock service
|
|
98
|
+
|
|
99
|
+
The 30-day handoff and the `/peek` endpoint live in the lock service, so rebuild the Redis image or redeploy the Worker along with the dario upgrade. In either order nothing breaks: a new dario against an old service gets a 404 from `/peek` and treats it as nothing to adopt, and an old dario against a new service still gets the longer handoff on `/acquire`.
|
|
94
100
|
|
|
95
101
|
---
|
|
96
102
|
|
|
@@ -111,6 +117,34 @@ The proxy says what it is doing at startup (`Pool shared state: on (instance …
|
|
|
111
117
|
|
|
112
118
|
It fails open like the lock: an unreachable service is one log line per outage and each instance carries on with its own state until the service answers again.
|
|
113
119
|
|
|
120
|
+
## Sharing the account list
|
|
121
|
+
|
|
122
|
+
Without it, each instance serves the accounts in its own `~/.dario/accounts/`. With `--pool-shared-accounts` the instances keep one account list in the same lock service: a replica that starts with an empty accounts directory (a second pod on a fresh volume) starts with its peers' accounts, and an account added or removed on one instance is added or removed on all of them.
|
|
123
|
+
|
|
124
|
+
```
|
|
125
|
+
DARIO_REFRESH_LOCK_URL=http://<lock-host>:8080
|
|
126
|
+
DARIO_REFRESH_LOCK_TOKEN=<shared secret>
|
|
127
|
+
DARIO_POOL_SHARED_ACCOUNTS=1 # or: dario proxy --pool-shared-accounts
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
Set it on every instance. It works with or without `--pool-shared-state`.
|
|
131
|
+
|
|
132
|
+
- **At startup**, before the pool loads, an instance reads the registry and merges it with its accounts directory. Per alias the newer record wins: a newer login first, then, for the same login, the later expiry. Accounts only this instance holds are published; accounts only the registry holds are written to this instance's accounts directory. The pool then loads from that directory as usual, so `/health` is ready as soon as a peer has an account.
|
|
133
|
+
- **Every 15 seconds** after that it merges again, and updates the running pool too: a new account joins it, newer credentials replace older ones, and a removed account leaves it. A `dario accounts add`, an admin API login or a token refresh on one instance reaches the others within one sync. An admin API login or removal is published straight away, and the other instances apply it on their next sync.
|
|
134
|
+
- **A removal leaves a marker** in the registry. The marker beats every login made before the removal, so an instance that was offline and still holds the old file deletes it instead of publishing it back. That holds for the `login` account taken from `credentials.json` at startup too: an instance whose login was removed starts without it instead of serving it. A login made after the removal beats the marker, and the account comes back everywhere.
|
|
135
|
+
- **`dario accounts remove <alias>`** publishes the removal first and deletes the file only once the service has taken it. If the service cannot be reached it removes nothing and says so, since the other instances would put the account straight back. A login made after the removal stays, in the registry or in the file here, and the command says so; run it again to remove that login too. If the file here can't be deleted, the command says so and exits non-zero, and this instance keeps serving the account until the file goes.
|
|
136
|
+
|
|
137
|
+
The proxy says it is on at startup (`Pool shared accounts: on (via …, syncing every 15s; fails open)`), and `GET /accounts` carries a `sharedAccounts` block: the last sync and the last good one, how many accounts and removal markers the registry holds, the files this instance saved and removed from it, the accounts and removals it published, and errors. Each change is logged: `added account "work" (wo***@example.com) from the registry`, `removed account "spare"; it was removed on another instance`, `published account "home" to the registry`.
|
|
138
|
+
|
|
139
|
+
Before turning it on:
|
|
140
|
+
|
|
141
|
+
- **The registry holds every seat's credentials, refresh tokens included, and they do not expire there.** That is what lets a new replica start with no files. Guard the lock service like the accounts directory itself: a real random `DARIO_REFRESH_LOCK_TOKEN`, the service on a private network, and a Redis that nothing else uses. Anyone who can read that Redis or the Durable Object can use every subscription in the pool.
|
|
142
|
+
- **Aliases are one namespace across all instances.** Two instances that each log a different subscription in under the same alias (`login`, most often) keep only the newer login, and the instance whose account was replaced logs it loudly: `account "login" here was ... and is now ...; aliases are shared across instances, so give each subscription its own alias`. Give each subscription its own alias.
|
|
143
|
+
- **Clocks have to agree.** The newer login and removal rules compare wall-clock times taken on different machines. Run NTP; Kubernetes nodes do by default.
|
|
144
|
+
- **Claude seats only.** ChatGPT accounts stay per instance, and the switch is off under `--no-claude-auth`.
|
|
145
|
+
- **It fails open.** With the service unreachable, each instance serves the accounts on its own disk and logs once per outage. A removal made while it is down is published when it answers again; if the instance restarts before then, that removal is lost and the account comes back from the registry, so remove it again.
|
|
146
|
+
- **Upgrade the lock service.** The `/accounts/*` endpoints are new: rebuild the Redis image or redeploy the Worker. An older service answers 404, which an instance logs as the service being unreachable and then serves its own disk, so nothing breaks, but nothing is shared either.
|
|
147
|
+
|
|
114
148
|
## Proving it works on your own setup
|
|
115
149
|
|
|
116
150
|
Do not take the above on trust. `test/integration/dual-instance-race.mjs` runs the real scenario: **two genuinely separate `node` processes**, each with its own isolated `~/.dario`, sharing nothing but the lock service, both racing to refresh the same account at the same instant.
|
|
@@ -136,7 +170,7 @@ Anthropic's token endpoint is the one mocked part, and only that — hammering t
|
|
|
136
170
|
|
|
137
171
|
## If you need true HA
|
|
138
172
|
|
|
139
|
-
Everything above is as far as dario goes today: shared credentials through the lock,
|
|
173
|
+
Everything above is as far as dario goes today: shared credentials through the lock, shared rate-limit readings and sticky bindings through `--pool-shared-state`, and a shared account list through `--pool-shared-accounts`, all eventually consistent and all failing open. Without `--pool-shared-accounts` the account list is per instance too, and copying the account files to every instance is safe with the refresh lock on: the copies share one refresh token, and whichever instance refreshes first hands the result to the others ([#1733](https://github.com/askalf/dario/issues/1733)). Without the lock, copies race exactly as described in section 1. Three things are still per instance:
|
|
140
174
|
|
|
141
175
|
- **ChatGPT seats.** The Codex accounts have their own refresh flow and usage windows; neither the lock nor the shared state covers them.
|
|
142
176
|
- **The ledger.** Each instance writes its own file, so `dario usage` and `/analytics` show that instance's share of the spend.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@askalf/dario",
|
|
3
|
-
"version": "6.
|
|
3
|
+
"version": "6.19.1",
|
|
4
4
|
"description": "Use your Claude and ChatGPT subscriptions in Cursor, Cline, Aider, Claude Code and the Agent SDK at subscription pricing, not per-token API bills. One local Anthropic + OpenAI-compatible endpoint: either plan answers either wire shape, opt-in failover when one hits its limit, and a ledger of what each plan absorbed.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|