@gotcos/glasses-server 6.79.2 → 6.80.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +9 -0
- package/CHANGELOG.md +19 -0
- package/README.md +10 -0
- package/managed-runtime-contract.json +1 -0
- package/package.json +9 -1
- package/server/index.ts +11 -0
- package/server/lib/agent-session-search.ts +8 -5
- package/server/lib/agent-session-store.ts +45 -4
- package/server/lib/archive.ts +2 -1
- package/server/lib/attached-provider-adapter.ts +11 -0
- package/server/lib/claude-bridge.ts +112 -8
- package/server/lib/claude-settings-guard.ts +94 -0
- package/server/lib/claude-steer.ts +27 -0
- package/server/lib/codex-bridge.ts +205 -32
- package/server/lib/codex-engine-sessions.ts +27 -6
- package/server/lib/codex-homes.ts +134 -0
- package/server/lib/codex-live-queue.ts +14 -0
- package/server/lib/codex-profile-home.ts +452 -0
- package/server/lib/codex-queue-control.ts +17 -1
- package/server/lib/codex-subagent-activity.ts +22 -0
- package/server/lib/conversation.ts +92 -15
- package/server/lib/direct-bridge.ts +853 -0
- package/server/lib/direct-sse.ts +262 -0
- package/server/lib/fork-thread.ts +11 -0
- package/server/lib/key-answer-fields.ts +49 -0
- package/server/lib/key-catalog.ts +301 -0
- package/server/lib/key-lanes-pricing.ts +12 -0
- package/server/lib/key-lanes-wiring.ts +346 -0
- package/server/lib/key-lanes.ts +145 -0
- package/server/lib/key-ledger.ts +123 -0
- package/server/lib/key-notices.ts +486 -0
- package/server/lib/key-profile-rejects.ts +25 -0
- package/server/lib/key-profile-routing.ts +173 -0
- package/server/lib/key-profiles.ts +182 -0
- package/server/lib/key-relay-contract.ts +262 -0
- package/server/lib/key-relay-errors.ts +211 -0
- package/server/lib/key-relay-meter.ts +303 -0
- package/server/lib/key-relay.ts +1018 -0
- package/server/lib/key-secrets-guard.ts +52 -0
- package/server/lib/key-store.ts +228 -0
- package/server/lib/keyed-sessions.ts +154 -0
- package/server/lib/lane-b-env.ts +169 -0
- package/server/lib/model-router.ts +96 -7
- package/server/lib/morning-brief-config.ts +3 -0
- package/server/lib/native-head.ts +13 -1
- package/server/lib/occupancy-probes.ts +4 -0
- package/server/lib/ollama-run-ledger.ts +96 -13
- package/server/lib/prompt-edit.ts +2 -0
- package/server/lib/query-job-coordinator.ts +2 -1
- package/server/lib/query-job-runtime.ts +67 -15
- package/server/lib/query-job-store.ts +7 -1
- package/server/lib/query-job-types.ts +15 -1
- package/server/lib/session-hook-events.ts +28 -0
- package/server/lib/session-hook-ledger.ts +10 -0
- package/server/lib/thread-occupancy.ts +22 -1
- package/server/lib/token-audit.ts +18 -4
- package/server/lib/work-handoff-requests.ts +3 -0
- package/server/routes/agent-session-bindings.ts +15 -0
- package/server/routes/agent-sessions.ts +17 -5
- package/server/routes/handoffs.ts +22 -1
- package/server/routes/health.ts +5 -0
- package/server/routes/key-lanes.ts +442 -0
- package/server/routes/openai-compat.ts +28 -3
- package/server/routes/query.ts +20 -6
- package/shared/key-preference.ts +57 -0
- package/shared/model-preference.ts +20 -2
|
@@ -0,0 +1,1018 @@
|
|
|
1
|
+
// Key lanes K1 P2 (plan v4 section 4): the loopback metering relay.
|
|
2
|
+
//
|
|
3
|
+
// The ONLY place on the Mac a real provider key is used. A keyed child (Claude Code, Codex) is
|
|
4
|
+
// pointed at `http://127.0.0.1:<ephemeral>/relay/{a,o}` and given a per-turn nonce instead of the
|
|
5
|
+
// key. For each request the relay checks the nonce, applies the profile's model policy, reserves
|
|
6
|
+
// against the key's daily cap, forwards with the real key, streams the answer back byte for byte
|
|
7
|
+
// while teeing a copy to a usage parser, and settles the measured cost.
|
|
8
|
+
//
|
|
9
|
+
// Bind 127.0.0.1 only, port 0. Never 3141 or 3143 (those face Tailscale), never 0.0.0.0.
|
|
10
|
+
// Start lazily, by the first issueTurn(). With no keyed turn there is no listener, no
|
|
11
|
+
// socket and no timer. close() when keyLanesActive() turns false.
|
|
12
|
+
// Secret read through readKeySecretForRelay() at request time, put in the upstream headers,
|
|
13
|
+
// and nowhere else. Upstream error bodies are scrubbed of it before they are passed on.
|
|
14
|
+
// Retries none here (no double billing). The child retries 429, 5xx and network errors.
|
|
15
|
+
// Breaker only 401, 402 and model-gone count (key-relay-errors.ts).
|
|
16
|
+
import http from 'node:http'
|
|
17
|
+
import type { IncomingMessage, ServerResponse } from 'node:http'
|
|
18
|
+
import type { AddressInfo } from 'node:net'
|
|
19
|
+
import { createHash, randomBytes } from 'node:crypto'
|
|
20
|
+
import { keyLanesActive, onKeyLanesInactive, registerKeyLanesStatusProvider } from './key-lanes.js'
|
|
21
|
+
import { getPublicKey, readKeySecretForRelay } from './key-store.js'
|
|
22
|
+
import type {
|
|
23
|
+
BreakerScope,
|
|
24
|
+
BreakerState,
|
|
25
|
+
KeyKind,
|
|
26
|
+
KeyRef,
|
|
27
|
+
KeyRelay,
|
|
28
|
+
RelayEngine,
|
|
29
|
+
RelayFailureKind,
|
|
30
|
+
RelayIssueOptions,
|
|
31
|
+
RelayModelAction,
|
|
32
|
+
RelayNonceBinding,
|
|
33
|
+
RelayRevokeReason,
|
|
34
|
+
RelayTurnHandle,
|
|
35
|
+
} from './key-relay-contract.js'
|
|
36
|
+
import { KEY_REF_RE, RELAY_BASE_PATHS, RELAY_LIMITS, RELAY_PATHS, RELAY_UPSTREAMS } from './key-relay-contract.js'
|
|
37
|
+
import {
|
|
38
|
+
RelayBreakers,
|
|
39
|
+
breakerScopeFor,
|
|
40
|
+
classifyUpstreamStatus,
|
|
41
|
+
countsTowardBreaker,
|
|
42
|
+
endsTurn,
|
|
43
|
+
redactSecrets,
|
|
44
|
+
relayErrorBody,
|
|
45
|
+
} from './key-relay-errors.js'
|
|
46
|
+
import type { RelayLocalError } from './key-relay-errors.js'
|
|
47
|
+
import type { KeyRelayMeter, ModelPrice } from './key-relay-meter.js'
|
|
48
|
+
import { defaultPriceFor, getSharedKeyMeter, reserveWithinHeadroomUsd, tokensToUsd } from './key-relay-meter.js'
|
|
49
|
+
|
|
50
|
+
/** A request past the in-flight cap waits this long for a slot before the busy 429. */
|
|
51
|
+
const IN_FLIGHT_WAIT_MS = 120_000
|
|
52
|
+
|
|
53
|
+
/** Ended nonces remembered (hash only) so a late request reads why; oldest dropped first. */
|
|
54
|
+
const ENDED_NONCES_MAX = 256
|
|
55
|
+
|
|
56
|
+
/** The only address the relay ever listens on. */
|
|
57
|
+
export const RELAY_HOST = '127.0.0.1'
|
|
58
|
+
export const OPENROUTER_API_BASE = 'https://openrouter.ai/api/v1'
|
|
59
|
+
const USAGE_POLL_MS = 60_000
|
|
60
|
+
const UPSTREAM_HEADERS_TIMEOUT_MS = 120_000
|
|
61
|
+
const PREFLIGHT_TIMEOUT_MS = 10_000
|
|
62
|
+
const RECONCILE_DELAY_MS = 15_000
|
|
63
|
+
const ERROR_BODY_CAP_BYTES = 1024 * 1024
|
|
64
|
+
const JSON_USAGE_CAP_BYTES = 8 * 1024 * 1024
|
|
65
|
+
const SSE_BUFFER_CAP_CHARS = 1024 * 1024
|
|
66
|
+
const LOG_PATH_CHARS = 120
|
|
67
|
+
|
|
68
|
+
// ── Model policy (plan 4.4) ──────────────────────────────────────────────────
|
|
69
|
+
|
|
70
|
+
const CLAUDE_ALIASES = new Set(['opus', 'sonnet', 'haiku', 'fable'])
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Equal to the profile slug forwards. A Claude id or alias (`claude-*`, `opus`, `sonnet`, `haiku`,
|
|
74
|
+
* `fable`, anything ending `[1m]`) on a Claude profile, or a bare OpenAI id (`gpt-*`, `o<n>*`,
|
|
75
|
+
* `codex-*`) on a Codex profile, is rewritten to the slug: the sub-model, small-fast and subagent
|
|
76
|
+
* ids the CLI picks on its own. Anything else is refused.
|
|
77
|
+
*/
|
|
78
|
+
export function decideRelayModel(requested: unknown, allowedModel: string, engine: RelayEngine): { action: RelayModelAction; model: string; requested: string } {
|
|
79
|
+
if (typeof requested !== 'string' || !requested) return { action: 'reject', model: '', requested: '' }
|
|
80
|
+
if (requested === allowedModel) return { action: 'forward', model: allowedModel, requested }
|
|
81
|
+
if (engine === 'claude') {
|
|
82
|
+
if (/\[1m\]$/i.test(requested)) return { action: 'rewrite', model: allowedModel, requested }
|
|
83
|
+
if (/^claude-/i.test(requested) || CLAUDE_ALIASES.has(requested.toLowerCase())) return { action: 'rewrite', model: allowedModel, requested }
|
|
84
|
+
return { action: 'reject', model: requested, requested }
|
|
85
|
+
}
|
|
86
|
+
if (!requested.includes('/') && /^(gpt-|o\d|codex-)/i.test(requested)) return { action: 'rewrite', model: allowedModel, requested }
|
|
87
|
+
return { action: 'reject', model: requested, requested }
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Which key kinds and model families a Lane B engine may use (plan 4.4). */
|
|
91
|
+
export function laneBModelAllowed(engine: RelayEngine, kind: KeyKind, model: string): boolean {
|
|
92
|
+
if (engine === 'claude') {
|
|
93
|
+
if (kind === 'openrouter') return model.startsWith('anthropic/')
|
|
94
|
+
if (kind === 'anthropic') return /^claude-/i.test(model)
|
|
95
|
+
return false
|
|
96
|
+
}
|
|
97
|
+
if (kind === 'openrouter') return model.startsWith('openai/')
|
|
98
|
+
if (kind === 'openai') return !model.includes('/')
|
|
99
|
+
return false
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// ── Usage parsing (tee of the response) ──────────────────────────────────────
|
|
103
|
+
|
|
104
|
+
export interface ParsedUsage {
|
|
105
|
+
input: number
|
|
106
|
+
output: number
|
|
107
|
+
costUsd?: number
|
|
108
|
+
/** The final usage event arrived (message_stop / response.completed, or a whole JSON body). */
|
|
109
|
+
complete: boolean
|
|
110
|
+
sawUsage: boolean
|
|
111
|
+
/** Characters of generated text seen, for a partial settle when the stream was cut. */
|
|
112
|
+
deltaChars: number
|
|
113
|
+
/** Canary R9: the cached share of `input`, priced at its own rates (read 0.1x, write 1.25x). */
|
|
114
|
+
cacheRead?: number
|
|
115
|
+
cacheWrite?: number
|
|
116
|
+
/** The provider's message id from message_start (OpenRouter's is the generation id). */
|
|
117
|
+
messageId?: string
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** Anthropic prompt-cache prices relative to the input price (canary R9: cache reads dominate keyed turns). */
|
|
121
|
+
export const CACHE_READ_PRICE_FACTOR = 0.1
|
|
122
|
+
export const CACHE_WRITE_PRICE_FACTOR = 1.25
|
|
123
|
+
|
|
124
|
+
/** USD for counted tokens with the cached share of the input at its own rates. */
|
|
125
|
+
export function cachePricedUsd(t: { input: number; output: number; cacheRead?: number; cacheWrite?: number }, price: ModelPrice): number {
|
|
126
|
+
const read = Math.max(0, t.cacheRead ?? 0), write = Math.max(0, t.cacheWrite ?? 0)
|
|
127
|
+
const plain = Math.max(0, t.input - read - write)
|
|
128
|
+
return (plain * price.inUsdPerM + read * price.inUsdPerM * CACHE_READ_PRICE_FACTOR + write * price.inUsdPerM * CACHE_WRITE_PRICE_FACTOR + Math.max(0, t.output) * price.outUsdPerM) / 1e6
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function num(v: unknown): number {
|
|
132
|
+
return typeof v === 'number' && Number.isFinite(v) && v >= 0 ? v : 0
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function costOf(usage: Record<string, unknown> | undefined): number | undefined {
|
|
136
|
+
const c = usage?.cost
|
|
137
|
+
return typeof c === 'number' && Number.isFinite(c) && c >= 0 ? c : undefined
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
export class RelayUsageParser {
|
|
141
|
+
private readonly decoder = new TextDecoder()
|
|
142
|
+
private buffer = ''
|
|
143
|
+
private jsonBytes = 0
|
|
144
|
+
private jsonChunks: string[] = []
|
|
145
|
+
private dead = false
|
|
146
|
+
readonly usage: ParsedUsage = { input: 0, output: 0, complete: false, sawUsage: false, deltaChars: 0 }
|
|
147
|
+
|
|
148
|
+
constructor(private readonly engine: RelayEngine, private readonly sse: boolean) {}
|
|
149
|
+
|
|
150
|
+
push(chunk: Uint8Array): void {
|
|
151
|
+
if (this.dead) return
|
|
152
|
+
const text = this.decoder.decode(chunk, { stream: true })
|
|
153
|
+
if (!this.sse) {
|
|
154
|
+
this.jsonBytes += chunk.byteLength
|
|
155
|
+
if (this.jsonBytes > JSON_USAGE_CAP_BYTES) { this.dead = true; this.jsonChunks = []; return }
|
|
156
|
+
this.jsonChunks.push(text)
|
|
157
|
+
return
|
|
158
|
+
}
|
|
159
|
+
this.buffer += text
|
|
160
|
+
let idx: number
|
|
161
|
+
while ((idx = this.buffer.search(/\r?\n\r?\n/)) >= 0) {
|
|
162
|
+
const event = this.buffer.slice(0, idx)
|
|
163
|
+
const sep = /\r?\n\r?\n/.exec(this.buffer.slice(idx))
|
|
164
|
+
this.buffer = this.buffer.slice(idx + (sep ? sep[0].length : 2))
|
|
165
|
+
this.onEvent(event)
|
|
166
|
+
}
|
|
167
|
+
if (this.buffer.length > SSE_BUFFER_CAP_CHARS) { this.dead = true; this.buffer = '' }
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
finish(): ParsedUsage {
|
|
171
|
+
if (this.dead) return this.usage
|
|
172
|
+
if (!this.sse) {
|
|
173
|
+
this.jsonChunks.push(this.decoder.decode())
|
|
174
|
+
try {
|
|
175
|
+
const body = JSON.parse(this.jsonChunks.join('')) as { usage?: Record<string, unknown>; response?: { usage?: Record<string, unknown> } }
|
|
176
|
+
const u = body.usage ?? body.response?.usage
|
|
177
|
+
if (u && typeof u === 'object') this.applyUsage(u, true)
|
|
178
|
+
} catch { /* not JSON: no usage, the reservation settles in full */ }
|
|
179
|
+
return this.usage
|
|
180
|
+
}
|
|
181
|
+
if (this.buffer.trim()) this.onEvent(this.buffer)
|
|
182
|
+
this.buffer = ''
|
|
183
|
+
return this.usage
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
private applyUsage(u: Record<string, unknown>, final: boolean): void {
|
|
187
|
+
const input = num(u.input_tokens) + num(u.cache_creation_input_tokens) + num(u.cache_read_input_tokens)
|
|
188
|
+
if (input >= this.usage.input) {
|
|
189
|
+
this.usage.cacheRead = num(u.cache_read_input_tokens)
|
|
190
|
+
this.usage.cacheWrite = num(u.cache_creation_input_tokens)
|
|
191
|
+
}
|
|
192
|
+
this.usage.input = Math.max(this.usage.input, input)
|
|
193
|
+
this.usage.output = Math.max(this.usage.output, num(u.output_tokens))
|
|
194
|
+
const cost = costOf(u)
|
|
195
|
+
if (cost !== undefined) this.usage.costUsd = cost
|
|
196
|
+
this.usage.sawUsage = true
|
|
197
|
+
if (final) this.usage.complete = true
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
private onEvent(raw: string): void {
|
|
201
|
+
const data = raw.split(/\r?\n/).filter(l => l.startsWith('data:')).map(l => l.slice(5).trimStart()).join('\n')
|
|
202
|
+
if (!data || data === '[DONE]') return
|
|
203
|
+
let ev: Record<string, unknown>
|
|
204
|
+
try { ev = JSON.parse(data) as Record<string, unknown> } catch { return }
|
|
205
|
+
const type = ev.type
|
|
206
|
+
if (this.engine === 'claude') {
|
|
207
|
+
if (type === 'message_start') {
|
|
208
|
+
const m = ev.message as { id?: unknown; usage?: Record<string, unknown> } | undefined
|
|
209
|
+
if (typeof m?.id === 'string' && m.id.length <= 128) this.usage.messageId = m.id
|
|
210
|
+
const u = m?.usage
|
|
211
|
+
if (u) this.applyUsage(u, false)
|
|
212
|
+
} else if (type === 'message_delta') {
|
|
213
|
+
const u = ev.usage as Record<string, unknown> | undefined
|
|
214
|
+
if (u) this.applyUsage(u, false)
|
|
215
|
+
} else if (type === 'content_block_delta') {
|
|
216
|
+
const d = ev.delta as Record<string, unknown> | undefined
|
|
217
|
+
for (const k of ['text', 'partial_json', 'thinking']) if (typeof d?.[k] === 'string') this.usage.deltaChars += (d[k] as string).length
|
|
218
|
+
} else if (type === 'message_stop') {
|
|
219
|
+
this.usage.complete = this.usage.sawUsage
|
|
220
|
+
}
|
|
221
|
+
return
|
|
222
|
+
}
|
|
223
|
+
if (typeof type === 'string' && type.endsWith('.delta') && typeof ev.delta === 'string') {
|
|
224
|
+
this.usage.deltaChars += ev.delta.length
|
|
225
|
+
} else if (type === 'response.completed' || type === 'response.incomplete' || type === 'response.failed' || type === 'response.done') {
|
|
226
|
+
const u = (ev.response as { usage?: Record<string, unknown> } | undefined)?.usage
|
|
227
|
+
if (u) this.applyUsage(u, true)
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/** QA W2: the running cost of a reply mid-stream: input as reported (or the body estimate), output at least the text seen. */
|
|
233
|
+
export function streamedUsd(usage: ParsedUsage, price: ModelPrice, bodyBytes: number): { usd: number; tokens: { input: number; output: number } } {
|
|
234
|
+
const input = usage.input > 0 ? usage.input : Math.ceil(Math.max(0, bodyBytes) / 4)
|
|
235
|
+
const output = Math.max(usage.output, Math.ceil(usage.deltaChars / 4))
|
|
236
|
+
const tokens = { input: Math.round(input), output: Math.round(output) }
|
|
237
|
+
// Canary R9: cache reads and writes at their own rates, so a reply that fits is not cut.
|
|
238
|
+
const cached = usage.input > 0 ? { cacheRead: usage.cacheRead, cacheWrite: usage.cacheWrite } : {}
|
|
239
|
+
return { usd: cachePricedUsd({ ...tokens, ...cached }, price), tokens }
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/** USD for what was parsed, or null when no usage arrived (settle the full reservation). */
|
|
243
|
+
export function usageUsd(usage: ParsedUsage, price: ModelPrice): { usd: number; tokens: { input: number; output: number } } | null {
|
|
244
|
+
if (!usage.sawUsage) return null
|
|
245
|
+
const output = usage.complete ? usage.output : Math.max(usage.output, Math.ceil(usage.deltaChars / 4))
|
|
246
|
+
const tokens = { input: Math.round(usage.input), output: Math.round(output) }
|
|
247
|
+
if (usage.complete && usage.costUsd !== undefined) return { usd: usage.costUsd, tokens }
|
|
248
|
+
return { usd: cachePricedUsd({ ...tokens, cacheRead: usage.cacheRead, cacheWrite: usage.cacheWrite }, price), tokens }
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
// ── The relay ────────────────────────────────────────────────────────────────
|
|
252
|
+
|
|
253
|
+
export type KeyRelayRefusalKind = RelayFailureKind | 'lanes_inactive'
|
|
254
|
+
|
|
255
|
+
/** Thrown by issueTurn(): the turn must not spawn. P3, P4 map `kind` to the P8 lens line. */
|
|
256
|
+
export class KeyRelayRefusal extends Error {
|
|
257
|
+
constructor(readonly kind: KeyRelayRefusalKind, readonly code: string, message: string) {
|
|
258
|
+
super(message)
|
|
259
|
+
this.name = 'KeyRelayRefusal'
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/**
|
|
264
|
+
* QA N10 / U2: every way a keyed request went wrong, not only the ones that end the turn. A 429,
|
|
265
|
+
* 5xx or network error is retried by the child; reported anyway so a turn that still fails reads
|
|
266
|
+
* why. `nonce_*` is a request on a token the relay already ended; `upstream_refused` a 4xx with
|
|
267
|
+
* no row of its own (403 moderation, a plain 400).
|
|
268
|
+
*/
|
|
269
|
+
export type RelayTurnFailureKind = RelayFailureKind | 'nonce_expired' | 'nonce_revoked' | 'upstream_refused' | 'relay_busy'
|
|
270
|
+
|
|
271
|
+
export interface RelayTurnFailure {
|
|
272
|
+
turnId: string
|
|
273
|
+
sessionId: string
|
|
274
|
+
profileId: string
|
|
275
|
+
keyRef: KeyRef
|
|
276
|
+
kind: RelayTurnFailureKind
|
|
277
|
+
status: number
|
|
278
|
+
/** The plan 4.6 "ends the turn" rows (cap, 401, 402, model gone). */
|
|
279
|
+
endsTurn?: boolean
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
export interface KeyRelayDeps {
|
|
283
|
+
now?: () => number
|
|
284
|
+
isActive?: () => boolean
|
|
285
|
+
secretFor?: (ref: KeyRef) => string | null
|
|
286
|
+
keyKindOf?: (ref: KeyRef) => KeyKind | undefined
|
|
287
|
+
/** For the "reset on key change or successful validation" rule. */
|
|
288
|
+
keyValidatedAt?: (ref: KeyRef) => string | undefined
|
|
289
|
+
meter?: KeyRelayMeter
|
|
290
|
+
priceFor?: (model: string) => ModelPrice
|
|
291
|
+
upstreams?: Partial<Record<keyof typeof RELAY_UPSTREAMS, string>>
|
|
292
|
+
/** OpenRouter's key and generation API (usage_daily, pre-spawn check, reconcile). */
|
|
293
|
+
openrouterApiBase?: string
|
|
294
|
+
fetch?: typeof fetch
|
|
295
|
+
usagePollMs?: number
|
|
296
|
+
upstreamHeadersTimeoutMs?: number
|
|
297
|
+
/** How long a request past the in-flight cap waits for a slot (default 120 s). */
|
|
298
|
+
inFlightWaitMs?: number
|
|
299
|
+
/** Delay before the generation-cost reconcile; null turns it off. */
|
|
300
|
+
reconcileDelayMs?: number | null
|
|
301
|
+
log?: (line: string) => void
|
|
302
|
+
/** The 4.6 "ends the turn" rows, so the bridge can stop the child and show the lens line. */
|
|
303
|
+
onTurnFailure?: (failure: RelayTurnFailure) => void
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
interface TurnEntry {
|
|
307
|
+
hash: string
|
|
308
|
+
nonceHead: string
|
|
309
|
+
binding: RelayNonceBinding
|
|
310
|
+
keyKind: KeyKind
|
|
311
|
+
expiresAt: number
|
|
312
|
+
inFlight: number
|
|
313
|
+
controllers: Set<AbortController>
|
|
314
|
+
timer?: ReturnType<typeof setTimeout>
|
|
315
|
+
/** QA (canary, parallel subagents): requests past the in-flight cap wait here, oldest first. */
|
|
316
|
+
waiters: Array<(got: boolean) => void>
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
interface Route {
|
|
320
|
+
engine: RelayEngine
|
|
321
|
+
metered: boolean
|
|
322
|
+
countTokens: boolean
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
const ROUTES: Record<string, Route> = {
|
|
326
|
+
[RELAY_PATHS.anthropicMessages]: { engine: 'claude', metered: true, countTokens: false },
|
|
327
|
+
[RELAY_PATHS.anthropicCountTokens]: { engine: 'claude', metered: false, countTokens: true },
|
|
328
|
+
[RELAY_PATHS.openaiResponses]: { engine: 'codex', metered: true, countTokens: false },
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
const DROP_RESPONSE_HEADERS = new Set([
|
|
332
|
+
'set-cookie', 'set-cookie2', 'content-encoding', 'content-length', 'transfer-encoding', 'connection', 'keep-alive',
|
|
333
|
+
'upgrade', 'proxy-authenticate', 'proxy-connection', 'trailer',
|
|
334
|
+
])
|
|
335
|
+
const CLAUDE_PASS_HEADERS = ['accept', 'anthropic-version', 'anthropic-beta', 'user-agent']
|
|
336
|
+
const CODEX_PASS_HEADERS = ['accept', 'openai-beta', 'originator', 'session_id', 'conversation_id', 'version', 'user-agent']
|
|
337
|
+
|
|
338
|
+
function hashNonce(nonce: string): string {
|
|
339
|
+
return createHash('sha256').update(nonce).digest('hex')
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
function headerValue(v: string | string[] | undefined): string | undefined {
|
|
343
|
+
return Array.isArray(v) ? v[0] : v
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
function bearer(v: string | undefined): string | undefined {
|
|
347
|
+
const m = /^Bearer\s+(\S+)\s*$/i.exec(v ?? '')
|
|
348
|
+
return m ? m[1] : undefined
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
function stamp(secret: string, validatedAt: string | undefined): string {
|
|
352
|
+
return createHash('sha256').update(secret).update('\0').update(validatedAt ?? '').digest('hex')
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
export class KeyRelayServer implements KeyRelay {
|
|
356
|
+
private readonly now: () => number
|
|
357
|
+
private readonly isActive: () => boolean
|
|
358
|
+
private readonly secretFor: (ref: KeyRef) => string | null
|
|
359
|
+
private readonly keyKindOf: (ref: KeyRef) => KeyKind | undefined
|
|
360
|
+
private readonly keyValidatedAt: (ref: KeyRef) => string | undefined
|
|
361
|
+
private readonly meter: KeyRelayMeter
|
|
362
|
+
private readonly priceFor: (model: string) => ModelPrice
|
|
363
|
+
private readonly upstreams: Record<keyof typeof RELAY_UPSTREAMS, string>
|
|
364
|
+
private readonly openrouterApi: string
|
|
365
|
+
private readonly fetchImpl: typeof fetch
|
|
366
|
+
private readonly usagePollMs: number
|
|
367
|
+
private readonly headersTimeoutMs: number
|
|
368
|
+
private readonly reconcileDelayMs: number | null
|
|
369
|
+
private readonly inFlightWaitMs: number
|
|
370
|
+
private readonly log: (line: string) => void
|
|
371
|
+
private readonly onTurnFailure: (failure: RelayTurnFailure) => void
|
|
372
|
+
readonly breakers: RelayBreakers
|
|
373
|
+
|
|
374
|
+
private server: http.Server | null = null
|
|
375
|
+
private listening: Promise<void> | null = null
|
|
376
|
+
private readonly turns = new Map<string, TurnEntry>()
|
|
377
|
+
private readonly ended = new Map<string, { binding: RelayNonceBinding; reason: RelayRevokeReason }>()
|
|
378
|
+
private readonly byTurnId = new Map<string, string>()
|
|
379
|
+
private readonly usageReadAt = new Map<KeyRef, number>()
|
|
380
|
+
private readonly keyStamps = new Map<KeyRef, string>()
|
|
381
|
+
private readonly reconcileTimers = new Set<ReturnType<typeof setTimeout>>()
|
|
382
|
+
|
|
383
|
+
constructor(deps: KeyRelayDeps = {}) {
|
|
384
|
+
this.now = deps.now ?? Date.now
|
|
385
|
+
this.isActive = deps.isActive ?? (() => keyLanesActive())
|
|
386
|
+
this.secretFor = deps.secretFor ?? readKeySecretForRelay
|
|
387
|
+
this.keyKindOf = deps.keyKindOf ?? (ref => getPublicKey(ref)?.kind)
|
|
388
|
+
this.keyValidatedAt = deps.keyValidatedAt ?? (ref => getPublicKey(ref)?.validatedAt)
|
|
389
|
+
this.meter = deps.meter ?? getSharedKeyMeter()
|
|
390
|
+
this.priceFor = deps.priceFor ?? defaultPriceFor
|
|
391
|
+
this.upstreams = { ...RELAY_UPSTREAMS, ...(deps.upstreams ?? {}) }
|
|
392
|
+
this.openrouterApi = (deps.openrouterApiBase ?? OPENROUTER_API_BASE).replace(/\/+$/, '')
|
|
393
|
+
this.fetchImpl = deps.fetch ?? (((input, init) => fetch(input, init)) as typeof fetch)
|
|
394
|
+
this.usagePollMs = deps.usagePollMs ?? USAGE_POLL_MS
|
|
395
|
+
this.headersTimeoutMs = deps.upstreamHeadersTimeoutMs ?? UPSTREAM_HEADERS_TIMEOUT_MS
|
|
396
|
+
this.reconcileDelayMs = deps.reconcileDelayMs === undefined ? RECONCILE_DELAY_MS : deps.reconcileDelayMs
|
|
397
|
+
this.inFlightWaitMs = deps.inFlightWaitMs ?? IN_FLIGHT_WAIT_MS
|
|
398
|
+
this.log = deps.log ?? (line => console.info(line))
|
|
399
|
+
this.onTurnFailure = deps.onTurnFailure ?? (() => {})
|
|
400
|
+
this.breakers = new RelayBreakers(this.now, (scope, id) => this.onBreakerOpen(scope, id))
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
// ── KeyRelay contract ──
|
|
404
|
+
|
|
405
|
+
port(): number | null {
|
|
406
|
+
const addr = this.server?.address()
|
|
407
|
+
return addr && typeof addr === 'object' ? (addr as AddressInfo).port : null
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
/** The listener's bound address, or null (tests pin 127.0.0.1). */
|
|
411
|
+
address(): string | null {
|
|
412
|
+
const addr = this.server?.address()
|
|
413
|
+
return addr && typeof addr === 'object' ? (addr as AddressInfo).address : null
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
async issueTurn(binding: RelayNonceBinding, options: RelayIssueOptions): Promise<RelayTurnHandle> {
|
|
417
|
+
if (!binding || !KEY_REF_RE.test(binding.keyRef) || (binding.engine !== 'claude' && binding.engine !== 'codex')
|
|
418
|
+
|| typeof binding.allowedModel !== 'string' || !binding.allowedModel || typeof binding.turnId !== 'string' || !binding.turnId) {
|
|
419
|
+
throw new KeyRelayRefusal('model_not_allowed', 'bad_binding', 'The key turn is missing its profile details.')
|
|
420
|
+
}
|
|
421
|
+
if (!this.isActive()) throw new KeyRelayRefusal('lanes_inactive', 'lanes_inactive', 'Key lanes are off.')
|
|
422
|
+
const ref = binding.keyRef
|
|
423
|
+
const kind = this.keyKindOf(ref)
|
|
424
|
+
const secret = this.secretFor(ref)
|
|
425
|
+
if (!kind || !secret) throw new KeyRelayRefusal('key_rejected', 'key_missing', 'That key is not saved on this Mac.')
|
|
426
|
+
if (!laneBModelAllowed(binding.engine, kind, binding.allowedModel)) {
|
|
427
|
+
throw new KeyRelayRefusal('model_not_allowed', 'lane_b_model', 'This model runs only on the direct lane.')
|
|
428
|
+
}
|
|
429
|
+
this.checkKeyStamp(ref, secret)
|
|
430
|
+
if (this.breakers.isOpen('key', ref)) throw new KeyRelayRefusal('key_rejected', 'breaker_open', 'This key failed three times in a row. It is paused for an hour.')
|
|
431
|
+
if (this.breakers.isOpen('profile', binding.profileId)) throw new KeyRelayRefusal('model_gone', 'breaker_open', 'This model failed three times in a row. It is paused for an hour.')
|
|
432
|
+
if (kind === 'openrouter') await this.preflight(ref, secret)
|
|
433
|
+
if (this.meter.atCap(ref)) throw new KeyRelayRefusal('cap_refused', 'cap_reached', 'Daily cap reached for this key.')
|
|
434
|
+
|
|
435
|
+
await this.ensureListening()
|
|
436
|
+
const previous = this.byTurnId.get(binding.turnId)
|
|
437
|
+
if (previous) this.drop(previous, 'cancel')
|
|
438
|
+
|
|
439
|
+
const nonce = randomBytes(RELAY_LIMITS.nonceBytes).toString('base64url')
|
|
440
|
+
const hash = hashNonce(nonce)
|
|
441
|
+
const wallMaxMs = Number.isFinite(options?.wallMaxMs) && options.wallMaxMs > 0 ? options.wallMaxMs : 0
|
|
442
|
+
const entry: TurnEntry = {
|
|
443
|
+
hash,
|
|
444
|
+
nonceHead: nonce.slice(0, RELAY_LIMITS.nonceLogChars),
|
|
445
|
+
binding: { ...binding },
|
|
446
|
+
keyKind: kind,
|
|
447
|
+
expiresAt: this.now() + wallMaxMs + RELAY_LIMITS.wallMaxGraceMs,
|
|
448
|
+
inFlight: 0,
|
|
449
|
+
controllers: new Set(),
|
|
450
|
+
waiters: [],
|
|
451
|
+
}
|
|
452
|
+
this.turns.set(hash, entry)
|
|
453
|
+
this.byTurnId.set(binding.turnId, hash)
|
|
454
|
+
this.scheduleExpiry(entry)
|
|
455
|
+
const origin = `http://${RELAY_HOST}:${this.port()}`
|
|
456
|
+
this.log(`[key-relay] issued nonce=${entry.nonceHead} turn=${binding.turnId} engine=${binding.engine}`)
|
|
457
|
+
return {
|
|
458
|
+
nonce,
|
|
459
|
+
origin,
|
|
460
|
+
childClosed: () => {
|
|
461
|
+
const e = this.turns.get(hash)
|
|
462
|
+
if (!e) return
|
|
463
|
+
e.expiresAt = Math.min(e.expiresAt, this.now() + RELAY_LIMITS.childCloseGraceMs)
|
|
464
|
+
this.scheduleExpiry(e)
|
|
465
|
+
},
|
|
466
|
+
revoke: (reason: RelayRevokeReason) => this.drop(hash, reason),
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
revokeTurn(turnId: string, reason: RelayRevokeReason): void {
|
|
471
|
+
const hash = this.byTurnId.get(turnId)
|
|
472
|
+
if (hash) this.drop(hash, reason)
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
revokeAll(reason: RelayRevokeReason): void {
|
|
476
|
+
for (const hash of [...this.turns.keys()]) this.drop(hash, reason)
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
async close(): Promise<void> {
|
|
480
|
+
this.revokeAll('lanes_inactive')
|
|
481
|
+
for (const t of this.reconcileTimers) clearTimeout(t)
|
|
482
|
+
this.reconcileTimers.clear()
|
|
483
|
+
const server = this.server
|
|
484
|
+
this.server = null
|
|
485
|
+
this.listening = null
|
|
486
|
+
if (!server) return
|
|
487
|
+
await new Promise<void>(resolve => {
|
|
488
|
+
server.close(() => resolve())
|
|
489
|
+
server.closeAllConnections()
|
|
490
|
+
})
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
/** Wait for an in-flight slot on this nonce; true once one is handed over, false on timeout or revoke. */
|
|
494
|
+
private waitForSlot(entry: TurnEntry): Promise<boolean> {
|
|
495
|
+
return new Promise(resolve => {
|
|
496
|
+
let done = false
|
|
497
|
+
const finish = (got: boolean) => { if (done) return; done = true; clearTimeout(timer); resolve(got) }
|
|
498
|
+
const timer = setTimeout(() => {
|
|
499
|
+
const i = entry.waiters.indexOf(finish)
|
|
500
|
+
if (i >= 0) entry.waiters.splice(i, 1)
|
|
501
|
+
finish(false)
|
|
502
|
+
}, this.inFlightWaitMs)
|
|
503
|
+
timer.unref?.()
|
|
504
|
+
entry.waiters.push(finish)
|
|
505
|
+
})
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
/** QA W2: settled spend today (this reservation excluded) plus what this reply has streamed so far, against the cap. */
|
|
509
|
+
private streamPastCap(ref: KeyRef, reservationId: string, usage: ParsedUsage, price: ModelPrice, bodyBytes: number): boolean {
|
|
510
|
+
const cap = this.meter.capUsd(ref)
|
|
511
|
+
if (cap === null) return true
|
|
512
|
+
const others = this.meter.spentTodayUsd(ref) - (this.meter.reservedUsd(reservationId) ?? 0)
|
|
513
|
+
return others + streamedUsd(usage, price, bodyBytes).usd > cap
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
// ── Extras for the status route, Control's Reset button and tests ──
|
|
517
|
+
|
|
518
|
+
breakerState(scope: BreakerScope, id: string): BreakerState {
|
|
519
|
+
return this.breakers.state(scope, id)
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
resetBreaker(scope: BreakerScope, id: string): void {
|
|
523
|
+
this.breakers.reset(scope, id)
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
/** QA W5: every live nonce of one profile (turned off, changed or removed). Returns how many. */
|
|
527
|
+
revokeProfile(profileId: string, reason: RelayRevokeReason = 'cancel'): number {
|
|
528
|
+
let n = 0
|
|
529
|
+
for (const [hash, e] of [...this.turns]) if (e.binding.profileId === profileId) { this.drop(hash, reason); n++ }
|
|
530
|
+
return n
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
/** Live nonces (tests and diagnostics; never the nonce itself). */
|
|
534
|
+
liveTurnCount(): number {
|
|
535
|
+
return this.turns.size
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
// ── Internals ──
|
|
539
|
+
|
|
540
|
+
private drop(hash: string, reason: RelayRevokeReason): void {
|
|
541
|
+
const e = this.turns.get(hash)
|
|
542
|
+
if (!e) return
|
|
543
|
+
this.turns.delete(hash)
|
|
544
|
+
if (this.byTurnId.get(e.binding.turnId) === hash) this.byTurnId.delete(e.binding.turnId)
|
|
545
|
+
// Remembered (hash, binding and reason only) so a late request on it can say why it failed.
|
|
546
|
+
this.ended.set(hash, { binding: e.binding, reason })
|
|
547
|
+
while (this.ended.size > ENDED_NONCES_MAX) {
|
|
548
|
+
const oldest = this.ended.keys().next().value
|
|
549
|
+
if (oldest === undefined) break
|
|
550
|
+
this.ended.delete(oldest)
|
|
551
|
+
}
|
|
552
|
+
if (e.timer) clearTimeout(e.timer)
|
|
553
|
+
for (const c of e.controllers) c.abort()
|
|
554
|
+
e.controllers.clear()
|
|
555
|
+
for (const w of e.waiters.splice(0)) w(false)
|
|
556
|
+
this.log(`[key-relay] revoked nonce=${e.nonceHead} reason=${reason}`)
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
private scheduleExpiry(e: TurnEntry): void {
|
|
560
|
+
if (e.timer) clearTimeout(e.timer)
|
|
561
|
+
const ms = Math.max(0, e.expiresAt - this.now())
|
|
562
|
+
e.timer = setTimeout(() => this.drop(e.hash, 'expired'), ms)
|
|
563
|
+
e.timer.unref?.()
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
private onBreakerOpen(scope: BreakerScope, id: string): void {
|
|
567
|
+
this.log(`[key-relay] breaker open scope=${scope}`)
|
|
568
|
+
for (const [hash, e] of [...this.turns]) {
|
|
569
|
+
if ((scope === 'key' && e.binding.keyRef === id) || (scope === 'profile' && e.binding.profileId === id)) this.drop(hash, 'breaker_open')
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
/** A new secret or a fresh validation for the same ref resets its breakers (plan 4.6). */
|
|
574
|
+
private checkKeyStamp(ref: KeyRef, secret: string): void {
|
|
575
|
+
const s = stamp(secret, this.keyValidatedAt(ref))
|
|
576
|
+
const before = this.keyStamps.get(ref)
|
|
577
|
+
if (before !== undefined && before !== s) {
|
|
578
|
+
this.breakers.reset('key', ref)
|
|
579
|
+
for (const e of this.turns.values()) if (e.binding.keyRef === ref) this.breakers.reset('profile', e.binding.profileId)
|
|
580
|
+
}
|
|
581
|
+
this.keyStamps.set(ref, s)
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
private ensureListening(): Promise<void> {
|
|
585
|
+
if (this.listening) return this.listening
|
|
586
|
+
const server = http.createServer((req, res) => { void this.handle(req, res) })
|
|
587
|
+
this.server = server
|
|
588
|
+
this.listening = new Promise<void>((resolve, reject) => {
|
|
589
|
+
server.once('error', reject)
|
|
590
|
+
server.listen(0, RELAY_HOST, () => {
|
|
591
|
+
server.off('error', reject)
|
|
592
|
+
resolve()
|
|
593
|
+
})
|
|
594
|
+
})
|
|
595
|
+
return this.listening
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
/** OpenRouter GET /key: pre-spawn check (plan 7.2) and usage_daily (plan 4.5). */
|
|
599
|
+
private async readKey(ref: KeyRef, secret: string): Promise<{ status: number; data?: Record<string, unknown> } | null> {
|
|
600
|
+
try {
|
|
601
|
+
const r = await this.fetchImpl(`${this.openrouterApi}/key`, {
|
|
602
|
+
headers: { authorization: `Bearer ${secret}`, accept: 'application/json' },
|
|
603
|
+
signal: AbortSignal.timeout(PREFLIGHT_TIMEOUT_MS),
|
|
604
|
+
redirect: 'error',
|
|
605
|
+
})
|
|
606
|
+
if (!r.ok) { await r.body?.cancel().catch(() => {}); return { status: r.status } }
|
|
607
|
+
const body = await r.json().catch(() => null) as { data?: Record<string, unknown> } | null
|
|
608
|
+
this.usageReadAt.set(ref, this.now())
|
|
609
|
+
const daily = body?.data?.usage_daily
|
|
610
|
+
if (typeof daily === 'number') this.meter.observeUsageDaily(ref, daily)
|
|
611
|
+
return { status: r.status, ...(body?.data ? { data: body.data } : {}) }
|
|
612
|
+
} catch {
|
|
613
|
+
return null
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
private async preflight(ref: KeyRef, secret: string): Promise<void> {
|
|
618
|
+
const r = await this.readKey(ref, secret)
|
|
619
|
+
if (!r) return // Unreachable: let the turn try; the child sees 502 and retries.
|
|
620
|
+
if (r.status === 401 || r.status === 403) {
|
|
621
|
+
this.breakers.record('key', ref, 'key_rejected', false)
|
|
622
|
+
throw new KeyRelayRefusal('key_rejected', 'key_rejected', 'OpenRouter rejected this key.')
|
|
623
|
+
}
|
|
624
|
+
if (r.status === 402) {
|
|
625
|
+
this.breakers.record('key', ref, 'out_of_credit', false)
|
|
626
|
+
throw new KeyRelayRefusal('out_of_credit', 'out_of_credit', 'This OpenRouter key has no credit left.')
|
|
627
|
+
}
|
|
628
|
+
const data = r.data
|
|
629
|
+
if (!data) return
|
|
630
|
+
if (data.limit === null || data.limit === undefined) {
|
|
631
|
+
throw new KeyRelayRefusal('out_of_credit', 'no_credit_limit', 'Set a credit limit on this key at openrouter.ai first.')
|
|
632
|
+
}
|
|
633
|
+
if (typeof data.limit_remaining === 'number' && data.limit_remaining <= 0) {
|
|
634
|
+
this.breakers.record('key', ref, 'out_of_credit', false)
|
|
635
|
+
throw new KeyRelayRefusal('out_of_credit', 'out_of_credit', 'This OpenRouter key has no credit left.')
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
private async refreshUsageIfStale(ref: KeyRef, kind: KeyKind, secret: string): Promise<void> {
|
|
640
|
+
if (kind !== 'openrouter') return
|
|
641
|
+
const at = this.usageReadAt.get(ref)
|
|
642
|
+
if (at !== undefined && this.now() - at < this.usagePollMs) return
|
|
643
|
+
await this.readKey(ref, secret)
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
private upstreamUrl(entry: TurnEntry, route: Route, search: string): string {
|
|
647
|
+
let base: string
|
|
648
|
+
if (route.engine === 'claude') base = entry.keyKind === 'anthropic' ? this.upstreams.anthropicMessages : this.upstreams.openrouterMessages
|
|
649
|
+
else base = entry.keyKind === 'openai' ? this.upstreams.openaiResponses : this.upstreams.openrouterResponses
|
|
650
|
+
return `${base}${route.countTokens ? '/count_tokens' : ''}${search}`
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
private upstreamHeaders(req: IncomingMessage, entry: TurnEntry, secret: string): Record<string, string> {
|
|
654
|
+
const h: Record<string, string> = { 'content-type': 'application/json', 'accept-encoding': 'identity' }
|
|
655
|
+
for (const name of entry.binding.engine === 'claude' ? CLAUDE_PASS_HEADERS : CODEX_PASS_HEADERS) {
|
|
656
|
+
const v = headerValue(req.headers[name])
|
|
657
|
+
if (v) h[name] = v
|
|
658
|
+
}
|
|
659
|
+
if (entry.binding.engine === 'claude' && !h['anthropic-version']) h['anthropic-version'] = '2023-06-01'
|
|
660
|
+
if (entry.keyKind === 'anthropic') h['x-api-key'] = secret
|
|
661
|
+
else h.authorization = `Bearer ${secret}`
|
|
662
|
+
return h
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
private sendLocal(res: ServerResponse, engine: RelayEngine | 'unknown', error: RelayLocalError, extra: Record<string, string> = {}): void {
|
|
666
|
+
if (res.headersSent) { res.destroy(); return }
|
|
667
|
+
const { status, body } = relayErrorBody(engine, error)
|
|
668
|
+
res.writeHead(status, { 'content-type': 'application/json', ...extra })
|
|
669
|
+
res.end(body)
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
private fail(entry: TurnEntry | { binding: RelayNonceBinding }, kind: RelayTurnFailureKind, status: number): void {
|
|
673
|
+
const b = entry.binding
|
|
674
|
+
const ends = kind !== 'nonce_expired' && kind !== 'nonce_revoked' && kind !== 'upstream_refused' && kind !== 'relay_busy' && endsTurn(kind)
|
|
675
|
+
try { this.onTurnFailure({ turnId: b.turnId, sessionId: b.sessionId, profileId: b.profileId, keyRef: b.keyRef, kind, status, endsTurn: ends }) } catch { /* a listener never breaks the relay */ }
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
private async readBody(req: IncomingMessage): Promise<Buffer | null> {
|
|
679
|
+
const chunks: Buffer[] = []
|
|
680
|
+
let size = 0
|
|
681
|
+
for await (const chunk of req) {
|
|
682
|
+
size += (chunk as Buffer).length
|
|
683
|
+
if (size > RELAY_LIMITS.bodyCapBytes) return null
|
|
684
|
+
chunks.push(chunk as Buffer)
|
|
685
|
+
}
|
|
686
|
+
return Buffer.concat(chunks)
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
private async handle(req: IncomingMessage, res: ServerResponse): Promise<void> {
|
|
690
|
+
let route: Route | undefined
|
|
691
|
+
try {
|
|
692
|
+
const url = new URL(req.url ?? '/', 'http://relay.invalid')
|
|
693
|
+
const path = url.pathname
|
|
694
|
+
if (req.headers.origin !== undefined) { req.resume(); return this.sendLocal(res, 'unknown', 'browser_refused') }
|
|
695
|
+
if (path === RELAY_PATHS.health && req.method === 'GET') {
|
|
696
|
+
res.writeHead(200, { 'content-type': 'application/json' })
|
|
697
|
+
res.end('{"ok":true}')
|
|
698
|
+
return
|
|
699
|
+
}
|
|
700
|
+
// Claude Code sends `HEAD /api/hello` under its base URL once per turn (canary R12): a
|
|
701
|
+
// reachability probe. Answered empty, never metered, never logged as a 404.
|
|
702
|
+
if (req.method === 'HEAD' && path === `${RELAY_BASE_PATHS.claude}/api/hello`) {
|
|
703
|
+
req.resume()
|
|
704
|
+
res.writeHead(204)
|
|
705
|
+
res.end()
|
|
706
|
+
return
|
|
707
|
+
}
|
|
708
|
+
route = ROUTES[path]
|
|
709
|
+
if (!route || req.method !== 'POST') {
|
|
710
|
+
this.log(`[key-relay] 404 ${req.method ?? '?'} ${path.slice(0, LOG_PATH_CHARS)}`)
|
|
711
|
+
req.resume()
|
|
712
|
+
return this.sendLocal(res, route?.engine ?? (path.startsWith('/relay/o/') ? 'codex' : 'unknown'), 'not_found')
|
|
713
|
+
}
|
|
714
|
+
const token = bearer(headerValue(req.headers.authorization)) ?? (route.engine === 'claude' ? headerValue(req.headers['x-api-key']) : undefined)
|
|
715
|
+
const entry = token ? this.turns.get(hashNonce(token)) : undefined
|
|
716
|
+
if (entry && !this.isActive()) this.revokeAll('lanes_inactive')
|
|
717
|
+
if (!entry || !this.turns.has(entry.hash) || this.now() >= entry.expiresAt || entry.binding.engine !== route.engine) {
|
|
718
|
+
if (entry && this.now() >= entry.expiresAt) this.drop(entry.hash, 'expired')
|
|
719
|
+
// QA N10: a token this relay ended reports why to its turn (an unknown one, nothing).
|
|
720
|
+
const ended = token ? this.ended.get(hashNonce(token)) : undefined
|
|
721
|
+
if (ended) this.fail({ binding: ended.binding }, ended.reason === 'expired' ? 'nonce_expired' : 'nonce_revoked', 401)
|
|
722
|
+
req.resume()
|
|
723
|
+
return this.sendLocal(res, route.engine, 'nonce_invalid')
|
|
724
|
+
}
|
|
725
|
+
// Canary QA: parallel subagents past the cap wait for a slot (bounded) instead of a 429
|
|
726
|
+
// that ends the turn on the CLI's raw error. Past the wait, the 429 is reported to the turn
|
|
727
|
+
// as relay_busy, which reads as a key-notices line.
|
|
728
|
+
if (entry.inFlight >= RELAY_LIMITS.inFlightPerNonce) {
|
|
729
|
+
const got = await this.waitForSlot(entry)
|
|
730
|
+
if (!got) {
|
|
731
|
+
req.resume()
|
|
732
|
+
if (this.turns.has(entry.hash)) this.fail(entry, 'relay_busy', 429)
|
|
733
|
+
return this.sendLocal(res, route.engine, 'too_many_in_flight', { 'retry-after': '1' })
|
|
734
|
+
}
|
|
735
|
+
} else {
|
|
736
|
+
entry.inFlight += 1
|
|
737
|
+
}
|
|
738
|
+
try {
|
|
739
|
+
await this.forward(req, res, entry, route, url.search)
|
|
740
|
+
} finally {
|
|
741
|
+
// The slot passes straight to the oldest waiter, or is released.
|
|
742
|
+
const next = entry.waiters.shift()
|
|
743
|
+
if (next) next(true)
|
|
744
|
+
else entry.inFlight -= 1
|
|
745
|
+
}
|
|
746
|
+
} catch {
|
|
747
|
+
this.sendLocal(res, route?.engine ?? 'unknown', 'network_error')
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
private async forward(req: IncomingMessage, res: ServerResponse, entry: TurnEntry, route: Route, search: string): Promise<void> {
|
|
752
|
+
const raw = await this.readBody(req)
|
|
753
|
+
if (raw === null) {
|
|
754
|
+
res.setHeader('connection', 'close')
|
|
755
|
+
return this.sendLocal(res, route.engine, 'too_large')
|
|
756
|
+
}
|
|
757
|
+
let json: Record<string, unknown>
|
|
758
|
+
try {
|
|
759
|
+
const parsed = JSON.parse(raw.toString('utf8')) as unknown
|
|
760
|
+
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) throw new Error('not an object')
|
|
761
|
+
json = parsed as Record<string, unknown>
|
|
762
|
+
} catch {
|
|
763
|
+
return this.sendLocal(res, route.engine, 'bad_request')
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
const b = entry.binding
|
|
767
|
+
const decision = decideRelayModel(json.model, b.allowedModel, route.engine)
|
|
768
|
+
if (decision.action === 'reject') {
|
|
769
|
+
this.log(`[key-relay] model refused nonce=${entry.nonceHead}`)
|
|
770
|
+
return this.sendLocal(res, route.engine, 'model_not_allowed')
|
|
771
|
+
}
|
|
772
|
+
let body: Buffer = raw
|
|
773
|
+
if (decision.action === 'rewrite') {
|
|
774
|
+
json.model = b.allowedModel
|
|
775
|
+
body = Buffer.from(JSON.stringify(json))
|
|
776
|
+
// QA N6: the requested name comes from the child; only safe characters reach the log line.
|
|
777
|
+
this.log(`[key-relay] model rewritten from=${String(decision.requested).slice(0, 64).replace(/[^A-Za-z0-9._:/@\[\]-]/g, '?')} nonce=${entry.nonceHead}`)
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
const secret = this.secretFor(b.keyRef)
|
|
781
|
+
if (!secret || this.breakers.isOpen('key', b.keyRef) || this.breakers.isOpen('profile', b.profileId)) {
|
|
782
|
+
this.drop(entry.hash, secret ? 'breaker_open' : 'remove_all')
|
|
783
|
+
return this.sendLocal(res, route.engine, 'nonce_invalid')
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
const price = this.priceFor(b.allowedModel)
|
|
787
|
+
let reservationId: string | null = null
|
|
788
|
+
if (route.metered) {
|
|
789
|
+
await this.refreshUsageIfStale(b.keyRef, entry.keyKind, secret)
|
|
790
|
+
const maxTokens = route.engine === 'claude' ? json.max_tokens : json.max_output_tokens
|
|
791
|
+
// QA W2: the request's own max_tokens, within what the key has left today.
|
|
792
|
+
const estimateUsd = reserveWithinHeadroomUsd({ bodyBytes: body.length, maxTokens: typeof maxTokens === 'number' ? maxTokens : null, price, headroomUsd: this.meter.headroomUsd(b.keyRef) })
|
|
793
|
+
const r = estimateUsd === null
|
|
794
|
+
? { ok: false as const }
|
|
795
|
+
: this.meter.reserve({ keyRef: b.keyRef, turnId: b.turnId, estimateUsd, profile: b.profileId, lane: 'key' })
|
|
796
|
+
if (!r.ok) {
|
|
797
|
+
this.log(`[key-relay] cap refused nonce=${entry.nonceHead}`)
|
|
798
|
+
this.fail(entry, 'cap_refused', 402)
|
|
799
|
+
return this.sendLocal(res, route.engine, 'cap_refused')
|
|
800
|
+
}
|
|
801
|
+
reservationId = r.reservationId
|
|
802
|
+
}
|
|
803
|
+
|
|
804
|
+
const controller = new AbortController()
|
|
805
|
+
entry.controllers.add(controller)
|
|
806
|
+
let childGone = false
|
|
807
|
+
let cutAtCap = false
|
|
808
|
+
res.on('close', () => {
|
|
809
|
+
if (!res.writableFinished) { childGone = true; controller.abort() }
|
|
810
|
+
})
|
|
811
|
+
const headersTimer = setTimeout(() => controller.abort(), this.headersTimeoutMs)
|
|
812
|
+
headersTimer.unref?.()
|
|
813
|
+
|
|
814
|
+
try {
|
|
815
|
+
let upstream: Response
|
|
816
|
+
try {
|
|
817
|
+
upstream = await this.fetchImpl(this.upstreamUrl(entry, route, search), {
|
|
818
|
+
method: 'POST',
|
|
819
|
+
headers: this.upstreamHeaders(req, entry, secret),
|
|
820
|
+
body: body as unknown as BodyInit,
|
|
821
|
+
signal: controller.signal,
|
|
822
|
+
redirect: 'error',
|
|
823
|
+
})
|
|
824
|
+
} catch {
|
|
825
|
+
clearTimeout(headersTimer)
|
|
826
|
+
if (reservationId) this.meter.settle({ reservationId, usd: 0, tokens: { input: 0, output: 0 } })
|
|
827
|
+
if (childGone) return
|
|
828
|
+
this.log(`[key-relay] upstream unreachable nonce=${entry.nonceHead}`)
|
|
829
|
+
this.fail(entry, 'network_error', 502)
|
|
830
|
+
return this.sendLocal(res, route.engine, 'network_error')
|
|
831
|
+
}
|
|
832
|
+
clearTimeout(headersTimer)
|
|
833
|
+
|
|
834
|
+
const outHeaders: Record<string, string> = {}
|
|
835
|
+
upstream.headers.forEach((value, name) => {
|
|
836
|
+
if (!DROP_RESPONSE_HEADERS.has(name.toLowerCase())) outHeaders[name] = redactSecrets(value, [secret])
|
|
837
|
+
})
|
|
838
|
+
|
|
839
|
+
if (upstream.status >= 400) {
|
|
840
|
+
const text = await this.readBounded(upstream, ERROR_BODY_CAP_BYTES)
|
|
841
|
+
const kind = classifyUpstreamStatus(upstream.status, text)
|
|
842
|
+
if (reservationId) this.meter.settle({ reservationId, usd: 0, tokens: { input: 0, output: 0 } })
|
|
843
|
+
if (kind && countsTowardBreaker(kind)) {
|
|
844
|
+
const scope = breakerScopeFor(kind)
|
|
845
|
+
this.breakers.record(scope, scope === 'key' ? b.keyRef : b.profileId, kind, false)
|
|
846
|
+
}
|
|
847
|
+
this.fail(entry, kind ?? 'upstream_refused', upstream.status)
|
|
848
|
+
this.log(`[key-relay] upstream ${upstream.status} ${kind ?? 'passed'} nonce=${entry.nonceHead}`)
|
|
849
|
+
if (childGone) return
|
|
850
|
+
res.writeHead(upstream.status, outHeaders)
|
|
851
|
+
res.end(redactSecrets(text, [secret]))
|
|
852
|
+
return
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
this.breakers.record('key', b.keyRef, null, true)
|
|
856
|
+
this.breakers.record('profile', b.profileId, null, true)
|
|
857
|
+
const contentType = upstream.headers.get('content-type') ?? ''
|
|
858
|
+
const parser = new RelayUsageParser(route.engine, contentType.includes('text/event-stream'))
|
|
859
|
+
res.writeHead(upstream.status, outHeaders)
|
|
860
|
+
res.flushHeaders()
|
|
861
|
+
if (upstream.body) {
|
|
862
|
+
const reader = upstream.body.getReader()
|
|
863
|
+
try {
|
|
864
|
+
for (;;) {
|
|
865
|
+
const { done, value } = await reader.read()
|
|
866
|
+
if (done) break
|
|
867
|
+
parser.push(value)
|
|
868
|
+
if (childGone) break
|
|
869
|
+
// QA W2: stop a reply whose running cost would take the key past today's cap.
|
|
870
|
+
if (reservationId && this.streamPastCap(b.keyRef, reservationId, parser.usage, price, body.length)) {
|
|
871
|
+
cutAtCap = true
|
|
872
|
+
controller.abort()
|
|
873
|
+
break
|
|
874
|
+
}
|
|
875
|
+
if (!res.write(value)) {
|
|
876
|
+
await new Promise<void>(resolve => {
|
|
877
|
+
const go = () => { res.off('drain', go); res.off('close', go); resolve() }
|
|
878
|
+
res.on('drain', go)
|
|
879
|
+
res.on('close', go)
|
|
880
|
+
})
|
|
881
|
+
if (childGone) break
|
|
882
|
+
}
|
|
883
|
+
}
|
|
884
|
+
} catch {
|
|
885
|
+
// Upstream dropped or we aborted (child gone, revoke): settle what was seen.
|
|
886
|
+
} finally {
|
|
887
|
+
reader.releaseLock()
|
|
888
|
+
}
|
|
889
|
+
}
|
|
890
|
+
if (!childGone && !res.writableEnded) res.end()
|
|
891
|
+
if (!reservationId) return
|
|
892
|
+
if (cutAtCap) {
|
|
893
|
+
const streamed = streamedUsd(parser.usage, price, body.length)
|
|
894
|
+
// Canary R9: the provider's generation id (header, else OpenRouter's message id) so the
|
|
895
|
+
// estimate is reconciled to the real cost like any other request.
|
|
896
|
+
const cutGeneration = upstream.headers.get('x-generation-id') ?? (parser.usage.messageId?.startsWith('gen-') ? parser.usage.messageId : undefined)
|
|
897
|
+
this.meter.settle({ reservationId, usd: streamed.usd, tokens: streamed.tokens, ...(cutGeneration ? { generationId: cutGeneration } : {}) })
|
|
898
|
+
if (cutGeneration && entry.keyKind === 'openrouter') this.scheduleReconcile(entry, cutGeneration, streamed.usd)
|
|
899
|
+
this.log(`[key-relay] cap reached mid-reply, stream cut nonce=${entry.nonceHead}`)
|
|
900
|
+
this.fail(entry, 'cap_refused', 402)
|
|
901
|
+
return
|
|
902
|
+
}
|
|
903
|
+
const usage = usageUsd(parser.finish(), price)
|
|
904
|
+
const generationId = upstream.headers.get('x-generation-id') ?? undefined
|
|
905
|
+
const settledUsd = usage ? usage.usd : (this.meter.reservedUsd(reservationId) ?? 0)
|
|
906
|
+
if (usage) this.meter.settle({ reservationId, usd: usage.usd, tokens: usage.tokens, ...(generationId ? { generationId } : {}) })
|
|
907
|
+
else this.meter.settleReserved(reservationId, { input: 0, output: 0 }, generationId)
|
|
908
|
+
if (generationId && entry.keyKind === 'openrouter') this.scheduleReconcile(entry, generationId, settledUsd)
|
|
909
|
+
} finally {
|
|
910
|
+
clearTimeout(headersTimer)
|
|
911
|
+
entry.controllers.delete(controller)
|
|
912
|
+
}
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
private async readBounded(r: Response, cap: number): Promise<string> {
|
|
916
|
+
if (!r.body) return ''
|
|
917
|
+
const reader = r.body.getReader()
|
|
918
|
+
const chunks: Uint8Array[] = []
|
|
919
|
+
let size = 0
|
|
920
|
+
try {
|
|
921
|
+
for (;;) {
|
|
922
|
+
const { done, value } = await reader.read()
|
|
923
|
+
if (done) break
|
|
924
|
+
size += value.byteLength
|
|
925
|
+
if (size > cap) { await reader.cancel().catch(() => {}); break }
|
|
926
|
+
chunks.push(value)
|
|
927
|
+
}
|
|
928
|
+
} catch { /* partial body */ }
|
|
929
|
+
return Buffer.concat(chunks.map(c => Buffer.from(c))).toString('utf8')
|
|
930
|
+
}
|
|
931
|
+
|
|
932
|
+
/** Plan 4.5 reconcile from OpenRouter's generation cost, once, after a delay. */
|
|
933
|
+
private scheduleReconcile(entry: TurnEntry, generationId: string, settledUsd: number): void {
|
|
934
|
+
if (this.reconcileDelayMs === null) return
|
|
935
|
+
const b = { ...entry.binding }
|
|
936
|
+
const timer = setTimeout(() => {
|
|
937
|
+
this.reconcileTimers.delete(timer)
|
|
938
|
+
void (async () => {
|
|
939
|
+
const secret = this.secretFor(b.keyRef)
|
|
940
|
+
if (!secret) return
|
|
941
|
+
try {
|
|
942
|
+
const r = await this.fetchImpl(`${this.openrouterApi}/generation?id=${encodeURIComponent(generationId)}`, {
|
|
943
|
+
headers: { authorization: `Bearer ${secret}`, accept: 'application/json' },
|
|
944
|
+
signal: AbortSignal.timeout(PREFLIGHT_TIMEOUT_MS),
|
|
945
|
+
redirect: 'error',
|
|
946
|
+
})
|
|
947
|
+
if (!r.ok) { await r.body?.cancel().catch(() => {}); return }
|
|
948
|
+
const body = await r.json() as { data?: { total_cost?: unknown; native_tokens_prompt?: unknown; native_tokens_completion?: unknown } }
|
|
949
|
+
const total = body?.data?.total_cost
|
|
950
|
+
if (typeof total !== 'number' || !Number.isFinite(total) || total < 0) return
|
|
951
|
+
const delta = total - settledUsd
|
|
952
|
+
this.meter.reconcile({
|
|
953
|
+
keyRef: b.keyRef, turnId: b.turnId, profile: b.profileId, lane: 'key', generationId, deltaUsd: delta,
|
|
954
|
+
tokens: { input: num(body.data?.native_tokens_prompt), output: num(body.data?.native_tokens_completion) },
|
|
955
|
+
})
|
|
956
|
+
} catch { /* reconcile is best effort; the settle row stands */ }
|
|
957
|
+
})()
|
|
958
|
+
}, this.reconcileDelayMs)
|
|
959
|
+
timer.unref?.()
|
|
960
|
+
this.reconcileTimers.add(timer)
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
|
|
964
|
+
export function createKeyRelay(deps: KeyRelayDeps = {}): KeyRelayServer {
|
|
965
|
+
return new KeyRelayServer(deps)
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
// ── The shared relay and its seams ───────────────────────────────────────────
|
|
969
|
+
|
|
970
|
+
let shared: KeyRelayServer | null = null
|
|
971
|
+
let seamsInstalled = false
|
|
972
|
+
const CLOSED_BREAKER: BreakerState = { open: false, failures: 0 }
|
|
973
|
+
|
|
974
|
+
/**
|
|
975
|
+
* Register the relay's answers for `GET /api/key-lanes/status` and its "lanes went inactive" hook.
|
|
976
|
+
* Map writes only: no file, socket or timer. Called by getKeyRelay(); P11 may also call it at boot
|
|
977
|
+
* behind keyLanesEnabled() so the status route shows today's spend before the first keyed turn.
|
|
978
|
+
*/
|
|
979
|
+
export function installKeyRelaySeams(): void {
|
|
980
|
+
if (seamsInstalled) return
|
|
981
|
+
seamsInstalled = true
|
|
982
|
+
registerKeyLanesStatusProvider({
|
|
983
|
+
spentTodayUsd: ref => getSharedKeyMeter().spentTodayUsd(ref),
|
|
984
|
+
breaker: ref => shared?.breakerState('key', ref) ?? CLOSED_BREAKER,
|
|
985
|
+
})
|
|
986
|
+
onKeyLanesInactive('key-relay', async reason => {
|
|
987
|
+
const relay = shared
|
|
988
|
+
if (!relay) return
|
|
989
|
+
relay.revokeAll(reason === 'remove_all' ? 'remove_all' : 'lanes_inactive')
|
|
990
|
+
if (reason === 'remove_all') getSharedKeyMeter().clear()
|
|
991
|
+
await relay.close()
|
|
992
|
+
})
|
|
993
|
+
}
|
|
994
|
+
|
|
995
|
+
/**
|
|
996
|
+
* The one relay for this server process (P3, P4 call this on a keyed turn). No listener until
|
|
997
|
+
* issueTurn. `deps` apply only when this call creates it (tests, P11 wiring).
|
|
998
|
+
*/
|
|
999
|
+
export function getKeyRelay(deps?: KeyRelayDeps): KeyRelayServer {
|
|
1000
|
+
if (!shared) shared = createKeyRelay(deps)
|
|
1001
|
+
installKeyRelaySeams()
|
|
1002
|
+
return shared
|
|
1003
|
+
}
|
|
1004
|
+
|
|
1005
|
+
/**
|
|
1006
|
+
* The shared relay when one exists, else null. Routes use this so a Control action never
|
|
1007
|
+
* creates the relay (and its deps) early: with no relay there is no live nonce and no open breaker.
|
|
1008
|
+
*/
|
|
1009
|
+
export function existingKeyRelay(): KeyRelayServer | null {
|
|
1010
|
+
return shared
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
/** Tests only. */
|
|
1014
|
+
export async function __resetKeyRelayForTests(): Promise<void> {
|
|
1015
|
+
if (shared) await shared.close()
|
|
1016
|
+
shared = null
|
|
1017
|
+
seamsInstalled = false
|
|
1018
|
+
}
|