@danceiny/gotry 0.2.0-rc.32 → 0.2.0-rc.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/README.zh-CN.md +2 -2
- package/dist/capabilities/session/bridge-ledger.js +168 -67
- package/dist/capabilities/session/extension-bridge.js +140 -46
- package/dist/capabilities/session/gui-flight.js +281 -69
- package/dist/capabilities/session/gui-hotel.js +74 -17
- package/dist/capabilities/session-search.js +6 -5
- package/dist/scripts/booking-copilot-dsh-planner-proof-tests.js +129 -2
- package/dist/scripts/booking-copilot-dsh-readiness-proof.js +53 -5
- package/dist/scripts/booking-copilot-gateway-chat-adapter-tests.js +233 -0
- package/dist/scripts/booking-copilot-planner-live-probe.js +1 -0
- package/dist/scripts/booking-copilot-startup-proof-tests.js +1 -1
- package/dist/scripts/bridge-ledger-tests.js +20 -18
- package/dist/scripts/fixtures/tenant-runaway-worker-fixture.mjs +73 -0
- package/dist/scripts/mutation-check.js +153 -0
- package/dist/scripts/nightly-evidence.js +4 -111
- package/dist/scripts/session-tests.js +42 -8
- package/dist/scripts/tenant-bridge-org-tests.js +1264 -0
- package/dist/scripts/tenant-capacity-probe.js +263 -0
- package/dist/scripts/tenant-compute-isolation-tests.js +707 -0
- package/dist/scripts/tenant-egress-credentials-tests.js +822 -0
- package/dist/scripts/tenant-executor-binding-tests.js +203 -0
- package/dist/scripts/tenant-prompt-partition-tests.js +265 -0
- package/dist/scripts/tenant-seam-adversarial-tests.js +957 -0
- package/dist/scripts/tenant-seam-quota-tests.js +383 -0
- package/dist/scripts/tenant-seam-test-kit.js +292 -0
- package/dist/scripts/tenant-state-copies-tests.js +607 -0
- package/dist/scripts/tenant-state-drill-tests.js +298 -0
- package/dist/scripts/tenant-state-drill-worker.js +164 -0
- package/dist/scripts/tenant-state-durable-conformance.js +101 -0
- package/dist/scripts/tenant-state-handoff-tests.js +854 -0
- package/dist/scripts/tenant-state-lease-conformance.js +142 -0
- package/dist/scripts/tenant-state-lease-worker.js +28 -0
- package/dist/scripts/tenant-state-router-tests.js +464 -0
- package/dist/scripts/tenant-state-test-kit.js +92 -0
- package/dist/scripts/tenant-state-tests.js +564 -0
- package/dist/src/backend/modules/booking-executor.js +61 -6
- package/dist/src/backend/modules/session-search.js +136 -39
- package/dist/src/booking-surface/dsh-llm-gateway-chat.js +434 -0
- package/dist/src/booking-surface/dsh-planner.js +150 -19
- package/dist/src/booking-surface/index.js +1 -0
- package/dist/src/booking-surface/managed-dsh-run-port.js +121 -7
- package/dist/src/booking-surface/server.js +193 -80
- package/dist/src/booking-surface/startup.js +153 -0
- package/dist/src/booking-surface/tenancy.js +188 -0
- package/dist/src/index.js +31 -1
- package/dist/src/llm-price-table.js +148 -0
- package/dist/src/tenant-seam/index.js +8 -0
- package/dist/src/tenant-seam/llm-broker.js +508 -0
- package/dist/src/tenant-seam/process-governor.js +281 -0
- package/dist/src/tenant-seam/quotas.js +360 -0
- package/dist/src/tenant-seam/seam.js +92 -0
- package/dist/src/tenant-state/durable-store.js +158 -0
- package/dist/src/tenant-state/epoch-copies.js +422 -0
- package/dist/src/tenant-state/errors.js +108 -0
- package/dist/src/tenant-state/ids.js +101 -0
- package/dist/src/tenant-state/index.js +11 -0
- package/dist/src/tenant-state/lease-store.js +203 -0
- package/dist/src/tenant-state/tenant-lease.js +150 -0
- package/dist/src/tenant-state/tenant-router.js +650 -0
- package/extension/manifest.json +2 -2
- package/package.json +4 -1
- package/ts/capabilities/session/bridge-ledger.ts +192 -78
- package/ts/capabilities/session/extension-bridge.ts +176 -65
- package/ts/capabilities/session/gui-flight.ts +318 -88
- package/ts/capabilities/session/gui-hotel.ts +72 -18
- package/ts/capabilities/session-search.ts +17 -5
- package/ts/src/booking-surface/dsh-llm-gateway-chat.js +434 -0
- package/ts/src/booking-surface/dsh-planner.ts +169 -19
- package/ts/src/booking-surface/index.ts +1 -0
- package/ts/src/booking-surface/managed-dsh-run-port.ts +150 -7
- package/ts/src/booking-surface/server.ts +215 -84
- package/ts/src/booking-surface/startup.ts +155 -0
- package/ts/src/booking-surface/tenancy.ts +242 -0
- package/ts/src/index.ts +28 -2
- package/ts/src/llm-price-table.ts +180 -0
- package/ts/src/tenant-seam/index.ts +9 -0
- package/ts/src/tenant-seam/llm-broker.ts +520 -0
- package/ts/src/tenant-seam/process-governor.ts +343 -0
- package/ts/src/tenant-seam/quotas.ts +433 -0
- package/ts/src/tenant-seam/seam.ts +142 -0
- package/ts/src/tenant-state/durable-store.ts +171 -0
- package/ts/src/tenant-state/epoch-copies.ts +491 -0
- package/ts/src/tenant-state/errors.ts +128 -0
- package/ts/src/tenant-state/ids.ts +108 -0
- package/ts/src/tenant-state/index.ts +12 -0
- package/ts/src/tenant-state/lease-store.ts +266 -0
- package/ts/src/tenant-state/tenant-lease.ts +175 -0
- package/ts/src/tenant-state/tenant-router.ts +743 -0
|
@@ -0,0 +1,433 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Quotas and metering per tenant and per organization (docs/rfc/aaas-rfc.md §4.4, threats S-10 and S-11).
|
|
3
|
+
*
|
|
4
|
+
* One tenant must not be able to move another tenant's latency budget, and one organization must not be able to
|
|
5
|
+
* move another organization's. The meter therefore counts at two scopes (the ledger tenant and its organization) and
|
|
6
|
+
* a global concurrency cap, and refuses with a typed `QuotaExceededError` (HTTP 429) before the expensive work: a
|
|
7
|
+
* refused turn is turned away after authentication and before the request body is read, the lease is acquired or a
|
|
8
|
+
* planner is started.
|
|
9
|
+
*
|
|
10
|
+
* Dimensions: concurrent turns, turn rate, LLM tokens and cost (priced by the frozen ADR-20 table, in the same code
|
|
11
|
+
* the nightly evidence uses), bridge job rate, and ledger size. Windows are rolling, driven by an injected
|
|
12
|
+
* monotonic clock. The LLM budget is checked before a turn and charged after it, so it can be overshot by the
|
|
13
|
+
* turns in flight: a soft cap, by design. A planner invocation that reports no usage (or zero tokens) is charged the
|
|
14
|
+
* configured worst case instead of nothing, so a silent or zero-answering feed cannot make the budget unreachable.
|
|
15
|
+
*
|
|
16
|
+
* State is per replica. Tenant-affine routing gives a tenant one owner, so tenant quotas hold fleet-wide; an
|
|
17
|
+
* organization's quota is per replica until a shared counter is chosen (A3).
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { assertModelPriced, priceMeasuredUsage, type LlmPriceTable, type MeasuredLlmUsage } from '../llm-price-table.ts'
|
|
21
|
+
import { TenantStateError } from '../tenant-state/errors.ts'
|
|
22
|
+
import { assertOrganizationId } from '../tenant-state/ids.ts'
|
|
23
|
+
import { assertSeamTenant, seamTenantKey, type SeamTenant } from './seam.ts'
|
|
24
|
+
|
|
25
|
+
export type QuotaScope = 'tenant' | 'organization' | 'global'
|
|
26
|
+
export type QuotaDimension = 'concurrent_turns' | 'turn_rate' | 'llm_tokens' | 'llm_cost' | 'bridge_jobs' | 'ledger_size'
|
|
27
|
+
|
|
28
|
+
export const QUOTA_DIMENSIONS: readonly QuotaDimension[] = ['concurrent_turns', 'turn_rate', 'llm_tokens', 'llm_cost', 'bridge_jobs', 'ledger_size']
|
|
29
|
+
|
|
30
|
+
/** A turn that found every slot taken has no better hint than "soon". */
|
|
31
|
+
const CONCURRENCY_RETRY_AFTER_MS = 1_000
|
|
32
|
+
/** A full ledger frees up only by an operator action, so the hint is long. */
|
|
33
|
+
const LEDGER_RETRY_AFTER_MS = 3_600_000
|
|
34
|
+
const MICRO = 1_000_000
|
|
35
|
+
const SWEEP_EVERY = 1_000
|
|
36
|
+
|
|
37
|
+
/** Refused by a quota. Carries the narrowest scope that refused, the dimension and a retry hint; no personal data. */
|
|
38
|
+
export class QuotaExceededError extends TenantStateError {
|
|
39
|
+
readonly scope: QuotaScope
|
|
40
|
+
readonly dimension: QuotaDimension
|
|
41
|
+
readonly limit: number
|
|
42
|
+
readonly retryAfterMs: number
|
|
43
|
+
|
|
44
|
+
constructor(scope: QuotaScope, dimension: QuotaDimension, limit: number, retryAfterMs: number) {
|
|
45
|
+
super('GOTRY_QUOTA_EXCEEDED', `${scope} quota exceeded: ${dimension}`)
|
|
46
|
+
this.scope = scope
|
|
47
|
+
this.dimension = dimension
|
|
48
|
+
this.limit = limit
|
|
49
|
+
this.retryAfterMs = Math.max(1, Math.ceil(retryAfterMs))
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Whole seconds for a `Retry-After` header, at least one. */
|
|
53
|
+
get retryAfterSeconds(): number {
|
|
54
|
+
return Math.max(1, Math.ceil(this.retryAfterMs / 1_000))
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export interface RateLimit {
|
|
59
|
+
readonly limit: number
|
|
60
|
+
readonly windowMs: number
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export interface QuotaLimits {
|
|
64
|
+
readonly concurrentTurns: number
|
|
65
|
+
readonly turns: RateLimit
|
|
66
|
+
readonly llm: { readonly tokens: number; readonly costUsd: number; readonly windowMs: number }
|
|
67
|
+
readonly bridgeJobs: RateLimit
|
|
68
|
+
readonly ledgerBytes: number
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export interface QuotaPolicy {
|
|
72
|
+
readonly tenant: QuotaLimits
|
|
73
|
+
readonly organization: QuotaLimits
|
|
74
|
+
readonly globalConcurrentTurns: number
|
|
75
|
+
/** What one planner invocation that reported no usage is charged: the worst case, never zero. */
|
|
76
|
+
readonly unprovenInvocation: { readonly tokens: number; readonly costUsd: number }
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Starting values, not a decision: they are conservative and unreviewed, and a deployment sets its own.
|
|
81
|
+
* Each tenant cap sits below its organization's, and each organization's below the global one, so that no single
|
|
82
|
+
* tenant can fill an organization and no single organization can fill the service.
|
|
83
|
+
*/
|
|
84
|
+
export const DEFAULT_QUOTA_POLICY: QuotaPolicy = Object.freeze({
|
|
85
|
+
tenant: {
|
|
86
|
+
concurrentTurns: 2,
|
|
87
|
+
turns: { limit: 30, windowMs: 60_000 },
|
|
88
|
+
llm: { tokens: 400_000, costUsd: 2, windowMs: 3_600_000 },
|
|
89
|
+
bridgeJobs: { limit: 12, windowMs: 60_000 },
|
|
90
|
+
ledgerBytes: 64 * 1024 * 1024,
|
|
91
|
+
},
|
|
92
|
+
organization: {
|
|
93
|
+
concurrentTurns: 16,
|
|
94
|
+
turns: { limit: 600, windowMs: 60_000 },
|
|
95
|
+
llm: { tokens: 20_000_000, costUsd: 100, windowMs: 3_600_000 },
|
|
96
|
+
bridgeJobs: { limit: 120, windowMs: 60_000 },
|
|
97
|
+
ledgerBytes: 8 * 1024 * 1024 * 1024,
|
|
98
|
+
},
|
|
99
|
+
globalConcurrentTurns: 64,
|
|
100
|
+
unprovenInvocation: { tokens: 50_000, costUsd: 0.25 },
|
|
101
|
+
})
|
|
102
|
+
|
|
103
|
+
export interface TurnPermit {
|
|
104
|
+
/** Frees the concurrency slot. Idempotent. */
|
|
105
|
+
release(): void
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
export interface QuotaAccountSnapshot {
|
|
109
|
+
readonly key: string
|
|
110
|
+
readonly activeTurns: number
|
|
111
|
+
readonly turnsInWindow: number
|
|
112
|
+
readonly llmTokens: number
|
|
113
|
+
readonly llmCostUsd: number
|
|
114
|
+
readonly bridgeJobs: number
|
|
115
|
+
readonly ledgerBytes: number
|
|
116
|
+
readonly rejected: Readonly<Record<QuotaDimension, number>>
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export interface QuotaSnapshot {
|
|
120
|
+
readonly globalActiveTurns: number
|
|
121
|
+
readonly tenants: readonly QuotaAccountSnapshot[]
|
|
122
|
+
readonly organizations: readonly QuotaAccountSnapshot[]
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export interface TenantQuotasOptions {
|
|
126
|
+
readonly policy: QuotaPolicy
|
|
127
|
+
readonly priceTable: LlmPriceTable
|
|
128
|
+
/** The model the planner runs. One the frozen table does not price stops the service before any spend (ADR-20). */
|
|
129
|
+
readonly model: string
|
|
130
|
+
/** Monotonic milliseconds. Suites drive it by hand. */
|
|
131
|
+
readonly clock?: () => number
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** A rolling window of amounts: the sum of what was added in the last `windowMs`. */
|
|
135
|
+
class RollingSum {
|
|
136
|
+
private readonly entries: Array<[number, number]> = []
|
|
137
|
+
private head = 0
|
|
138
|
+
private total = 0
|
|
139
|
+
readonly windowMs: number
|
|
140
|
+
|
|
141
|
+
constructor(windowMs: number) {
|
|
142
|
+
this.windowMs = windowMs
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
private prune(now: number): void {
|
|
146
|
+
const horizon = now - this.windowMs
|
|
147
|
+
while (this.head < this.entries.length && this.entries[this.head]![0] <= horizon) {
|
|
148
|
+
this.total -= this.entries[this.head]![1]
|
|
149
|
+
this.head += 1
|
|
150
|
+
}
|
|
151
|
+
if (this.head > 1_024 && this.head * 2 > this.entries.length) {
|
|
152
|
+
this.entries.splice(0, this.head)
|
|
153
|
+
this.head = 0
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
sum(now: number): number {
|
|
158
|
+
this.prune(now)
|
|
159
|
+
return this.total
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
add(now: number, amount: number): void {
|
|
163
|
+
this.prune(now)
|
|
164
|
+
this.entries.push([now, amount])
|
|
165
|
+
this.total += amount
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** Milliseconds until the sum falls strictly below `limit` if nothing more is added; 0 when it already is. */
|
|
169
|
+
msUntilBelow(now: number, limit: number): number {
|
|
170
|
+
this.prune(now)
|
|
171
|
+
let remaining = this.total
|
|
172
|
+
if (remaining < limit) return 0
|
|
173
|
+
for (let index = this.head; index < this.entries.length; index += 1) {
|
|
174
|
+
remaining -= this.entries[index]![1]
|
|
175
|
+
if (remaining < limit) return Math.max(1, this.entries[index]![0] + this.windowMs - now)
|
|
176
|
+
}
|
|
177
|
+
return this.windowMs
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
idle(now: number): boolean {
|
|
181
|
+
this.prune(now)
|
|
182
|
+
return this.head === this.entries.length
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
class Account {
|
|
187
|
+
active = 0
|
|
188
|
+
readonly turns: RollingSum
|
|
189
|
+
readonly tokens: RollingSum
|
|
190
|
+
readonly costMicro: RollingSum
|
|
191
|
+
readonly bridge: RollingSum
|
|
192
|
+
ledgerBytes = 0
|
|
193
|
+
readonly rejected: Record<QuotaDimension, number> = { concurrent_turns: 0, turn_rate: 0, llm_tokens: 0, llm_cost: 0, bridge_jobs: 0, ledger_size: 0 }
|
|
194
|
+
|
|
195
|
+
constructor(limits: QuotaLimits) {
|
|
196
|
+
this.turns = new RollingSum(limits.turns.windowMs)
|
|
197
|
+
this.tokens = new RollingSum(limits.llm.windowMs)
|
|
198
|
+
this.costMicro = new RollingSum(limits.llm.windowMs)
|
|
199
|
+
this.bridge = new RollingSum(limits.bridgeJobs.windowMs)
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/** No live turn and nothing left in any window. The ledger size is not part of it: the sweep takes it out of the organization total. */
|
|
203
|
+
idle(now: number): boolean {
|
|
204
|
+
return this.active === 0 && this.turns.idle(now) && this.tokens.idle(now) && this.costMicro.idle(now) && this.bridge.idle(now)
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
function positiveInteger(name: string, value: unknown): void {
|
|
209
|
+
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 1) throw new RangeError(`${name} must be a positive integer`)
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function positiveNumber(name: string, value: unknown): void {
|
|
213
|
+
if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) throw new RangeError(`${name} must be a positive number`)
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
function validateLimits(scope: string, limits: QuotaLimits): void {
|
|
217
|
+
positiveInteger(`${scope}.concurrentTurns`, limits.concurrentTurns)
|
|
218
|
+
positiveInteger(`${scope}.turns.limit`, limits.turns.limit)
|
|
219
|
+
positiveInteger(`${scope}.turns.windowMs`, limits.turns.windowMs)
|
|
220
|
+
positiveInteger(`${scope}.llm.tokens`, limits.llm.tokens)
|
|
221
|
+
positiveNumber(`${scope}.llm.costUsd`, limits.llm.costUsd)
|
|
222
|
+
positiveInteger(`${scope}.llm.windowMs`, limits.llm.windowMs)
|
|
223
|
+
positiveInteger(`${scope}.bridgeJobs.limit`, limits.bridgeJobs.limit)
|
|
224
|
+
positiveInteger(`${scope}.bridgeJobs.windowMs`, limits.bridgeJobs.windowMs)
|
|
225
|
+
positiveInteger(`${scope}.ledgerBytes`, limits.ledgerBytes)
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
function validatePolicy(policy: QuotaPolicy): void {
|
|
229
|
+
validateLimits('tenant', policy.tenant)
|
|
230
|
+
validateLimits('organization', policy.organization)
|
|
231
|
+
positiveInteger('globalConcurrentTurns', policy.globalConcurrentTurns)
|
|
232
|
+
positiveInteger('unprovenInvocation.tokens', policy.unprovenInvocation.tokens)
|
|
233
|
+
positiveNumber('unprovenInvocation.costUsd', policy.unprovenInvocation.costUsd)
|
|
234
|
+
// A tenant cap above its organization's can never bind, and an organization above the global one lets it starve the rest.
|
|
235
|
+
if (policy.tenant.concurrentTurns > policy.organization.concurrentTurns) throw new RangeError('tenant.concurrentTurns must not exceed organization.concurrentTurns')
|
|
236
|
+
if (policy.organization.concurrentTurns > policy.globalConcurrentTurns) throw new RangeError('organization.concurrentTurns must not exceed globalConcurrentTurns')
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
export class TenantQuotas {
|
|
240
|
+
private readonly policy: QuotaPolicy
|
|
241
|
+
private readonly priceTable: LlmPriceTable
|
|
242
|
+
private readonly clock: () => number
|
|
243
|
+
private readonly tenants = new Map<string, Account>()
|
|
244
|
+
private readonly organizations = new Map<string, Account>()
|
|
245
|
+
private globalActive = 0
|
|
246
|
+
private admissions = 0
|
|
247
|
+
|
|
248
|
+
constructor(options: TenantQuotasOptions) {
|
|
249
|
+
validatePolicy(options.policy)
|
|
250
|
+
assertModelPriced(options.model, options.priceTable)
|
|
251
|
+
this.policy = options.policy
|
|
252
|
+
this.priceTable = options.priceTable
|
|
253
|
+
this.clock = options.clock ?? (() => performance.now())
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* Admits one turn or refuses it. Checks run narrowest scope first (tenant, organization, global) so a hot tenant is
|
|
258
|
+
* told it hit its own limit, and an innocent one only ever sees an organization limit that the sum really reached.
|
|
259
|
+
* Nothing is reserved unless every check passes.
|
|
260
|
+
*/
|
|
261
|
+
admitTurn(tenant: SeamTenant): TurnPermit {
|
|
262
|
+
const seam = assertSeamTenant(tenant)
|
|
263
|
+
const now = this.clock()
|
|
264
|
+
// Every attempt counts toward the sweep, so a flood of refused attempts cannot outrun it.
|
|
265
|
+
this.maybeSweep(now)
|
|
266
|
+
const key = seamTenantKey(seam)
|
|
267
|
+
// A tenant is registered only when a turn is admitted: a fresh tenant that is refused leaves nothing behind.
|
|
268
|
+
const known = this.tenants.get(key)
|
|
269
|
+
const tenantAccount = known ?? new Account(this.policy.tenant)
|
|
270
|
+
const organizationAccount = this.organizationAccount(seam.organization)
|
|
271
|
+
const { policy } = this
|
|
272
|
+
this.refuseIfOver(tenantAccount, 'tenant', 'concurrent_turns', tenantAccount.active, policy.tenant.concurrentTurns, CONCURRENCY_RETRY_AFTER_MS)
|
|
273
|
+
this.refuseIfOverRolling(tenantAccount, 'tenant', 'turn_rate', tenantAccount.turns, now, policy.tenant.turns.limit)
|
|
274
|
+
this.refuseIfOverRolling(tenantAccount, 'tenant', 'llm_tokens', tenantAccount.tokens, now, policy.tenant.llm.tokens)
|
|
275
|
+
this.refuseIfOverRolling(tenantAccount, 'tenant', 'llm_cost', tenantAccount.costMicro, now, Math.round(policy.tenant.llm.costUsd * MICRO), MICRO)
|
|
276
|
+
this.refuseIfOver(organizationAccount, 'organization', 'concurrent_turns', organizationAccount.active, policy.organization.concurrentTurns, CONCURRENCY_RETRY_AFTER_MS)
|
|
277
|
+
this.refuseIfOverRolling(organizationAccount, 'organization', 'turn_rate', organizationAccount.turns, now, policy.organization.turns.limit)
|
|
278
|
+
this.refuseIfOverRolling(organizationAccount, 'organization', 'llm_tokens', organizationAccount.tokens, now, policy.organization.llm.tokens)
|
|
279
|
+
this.refuseIfOverRolling(organizationAccount, 'organization', 'llm_cost', organizationAccount.costMicro, now, Math.round(policy.organization.llm.costUsd * MICRO), MICRO)
|
|
280
|
+
this.refuseIfOver(organizationAccount, 'global', 'concurrent_turns', this.globalActive, policy.globalConcurrentTurns, CONCURRENCY_RETRY_AFTER_MS)
|
|
281
|
+
if (!known) this.tenants.set(key, tenantAccount)
|
|
282
|
+
tenantAccount.active += 1
|
|
283
|
+
organizationAccount.active += 1
|
|
284
|
+
this.globalActive += 1
|
|
285
|
+
tenantAccount.turns.add(now, 1)
|
|
286
|
+
organizationAccount.turns.add(now, 1)
|
|
287
|
+
let released = false
|
|
288
|
+
return {
|
|
289
|
+
release: () => {
|
|
290
|
+
if (released) return
|
|
291
|
+
released = true
|
|
292
|
+
tenantAccount.active -= 1
|
|
293
|
+
organizationAccount.active -= 1
|
|
294
|
+
this.globalActive -= 1
|
|
295
|
+
},
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/**
|
|
300
|
+
* Charges measured usage, priced by the frozen table. Throws for an unpriced model, malformed counts or a report of zero
|
|
301
|
+
* tokens (no completion costs nothing, so a relay that answers zeros is not measuring); the caller then charges the worst case.
|
|
302
|
+
*/
|
|
303
|
+
recordLlmUsage(tenant: SeamTenant, model: string, usage: MeasuredLlmUsage): void {
|
|
304
|
+
const seam = assertSeamTenant(tenant)
|
|
305
|
+
const priced = priceMeasuredUsage(model, usage, this.priceTable)
|
|
306
|
+
if (priced.tokens === 0) throw new RangeError('a usage report of zero tokens proves nothing')
|
|
307
|
+
this.charge(seam, priced.tokens, Math.round(priced.costUsd * MICRO))
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/** Charges one planner invocation that reported no usable usage: the configured worst case. */
|
|
311
|
+
chargeUnprovenInvocation(tenant: SeamTenant): void {
|
|
312
|
+
const seam = assertSeamTenant(tenant)
|
|
313
|
+
this.charge(seam, this.policy.unprovenInvocation.tokens, Math.round(this.policy.unprovenInvocation.costUsd * MICRO))
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
/** Counts one bridge job for a tenant and its organization, or refuses it. */
|
|
317
|
+
admitBridgeJob(tenant: SeamTenant): void {
|
|
318
|
+
const seam = assertSeamTenant(tenant)
|
|
319
|
+
const now = this.clock()
|
|
320
|
+
const tenantAccount = this.tenantAccount(seamTenantKey(seam))
|
|
321
|
+
const organizationAccount = this.organizationAccount(seam.organization)
|
|
322
|
+
this.refuseIfOverRolling(tenantAccount, 'tenant', 'bridge_jobs', tenantAccount.bridge, now, this.policy.tenant.bridgeJobs.limit)
|
|
323
|
+
this.refuseIfOverRolling(organizationAccount, 'organization', 'bridge_jobs', organizationAccount.bridge, now, this.policy.organization.bridgeJobs.limit)
|
|
324
|
+
tenantAccount.bridge.add(now, 1)
|
|
325
|
+
organizationAccount.bridge.add(now, 1)
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/** The same for a caller that is known only by organization, such as the extension lane's HTTP surface. */
|
|
329
|
+
admitOrganizationBridgeJob(organization: string): void {
|
|
330
|
+
const now = this.clock()
|
|
331
|
+
const organizationAccount = this.organizationAccount(assertOrganizationId(organization))
|
|
332
|
+
this.refuseIfOverRolling(organizationAccount, 'organization', 'bridge_jobs', organizationAccount.bridge, now, this.policy.organization.bridgeJobs.limit)
|
|
333
|
+
organizationAccount.bridge.add(now, 1)
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
/**
|
|
337
|
+
* Records the size of a tenant's ledger and refuses while it is at its limit, or while the ledgers of the
|
|
338
|
+
* organization's tenants served here add up to the organization's limit.
|
|
339
|
+
*/
|
|
340
|
+
observeLedgerBytes(tenant: SeamTenant, bytes: number): void {
|
|
341
|
+
const seam = assertSeamTenant(tenant)
|
|
342
|
+
if (!Number.isSafeInteger(bytes) || bytes < 0) throw new RangeError('bytes must be a non-negative integer')
|
|
343
|
+
const tenantAccount = this.tenantAccount(seamTenantKey(seam))
|
|
344
|
+
const organizationAccount = this.organizationAccount(seam.organization)
|
|
345
|
+
organizationAccount.ledgerBytes += bytes - tenantAccount.ledgerBytes
|
|
346
|
+
tenantAccount.ledgerBytes = bytes
|
|
347
|
+
this.refuseIfOver(tenantAccount, 'tenant', 'ledger_size', bytes, this.policy.tenant.ledgerBytes, LEDGER_RETRY_AFTER_MS)
|
|
348
|
+
this.refuseIfOver(organizationAccount, 'organization', 'ledger_size', organizationAccount.ledgerBytes, this.policy.organization.ledgerBytes, LEDGER_RETRY_AFTER_MS)
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/** Forgets a tenant's counters (after a forget, or when its ledger leaves this replica). */
|
|
352
|
+
forgetTenant(tenant: SeamTenant): void {
|
|
353
|
+
const seam = assertSeamTenant(tenant)
|
|
354
|
+
const key = seamTenantKey(seam)
|
|
355
|
+
const account = this.tenants.get(key)
|
|
356
|
+
if (!account) return
|
|
357
|
+
const organizationAccount = this.organizations.get(seam.organization)
|
|
358
|
+
if (organizationAccount) organizationAccount.ledgerBytes -= account.ledgerBytes
|
|
359
|
+
this.tenants.delete(key)
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
/** Counters keyed by `<organization>/<hmac tenant id>`: nothing that names a person. */
|
|
363
|
+
snapshot(): QuotaSnapshot {
|
|
364
|
+
const now = this.clock()
|
|
365
|
+
const describe = (key: string, account: Account): QuotaAccountSnapshot => ({
|
|
366
|
+
key,
|
|
367
|
+
activeTurns: account.active,
|
|
368
|
+
turnsInWindow: account.turns.sum(now),
|
|
369
|
+
llmTokens: account.tokens.sum(now),
|
|
370
|
+
llmCostUsd: account.costMicro.sum(now) / MICRO,
|
|
371
|
+
bridgeJobs: account.bridge.sum(now),
|
|
372
|
+
ledgerBytes: account.ledgerBytes,
|
|
373
|
+
rejected: { ...account.rejected },
|
|
374
|
+
})
|
|
375
|
+
return {
|
|
376
|
+
globalActiveTurns: this.globalActive,
|
|
377
|
+
tenants: [...this.tenants].map(([key, account]) => describe(key, account)),
|
|
378
|
+
organizations: [...this.organizations].map(([key, account]) => describe(key, account)),
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
private charge(seam: SeamTenant, tokens: number, costMicro: number): void {
|
|
383
|
+
const now = this.clock()
|
|
384
|
+
for (const account of [this.tenantAccount(seamTenantKey(seam)), this.organizationAccount(seam.organization)]) {
|
|
385
|
+
account.tokens.add(now, tokens)
|
|
386
|
+
account.costMicro.add(now, costMicro)
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
private tenantAccount(key: string): Account {
|
|
391
|
+
let account = this.tenants.get(key)
|
|
392
|
+
if (!account) {
|
|
393
|
+
account = new Account(this.policy.tenant)
|
|
394
|
+
this.tenants.set(key, account)
|
|
395
|
+
}
|
|
396
|
+
return account
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
private organizationAccount(organization: string): Account {
|
|
400
|
+
let account = this.organizations.get(organization)
|
|
401
|
+
if (!account) {
|
|
402
|
+
account = new Account(this.policy.organization)
|
|
403
|
+
this.organizations.set(organization, account)
|
|
404
|
+
}
|
|
405
|
+
return account
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/** Refuses when `used` has reached `limit`. */
|
|
409
|
+
private refuseIfOver(account: Account, scope: QuotaScope, dimension: QuotaDimension, used: number, limit: number, retryAfterMs: number): void {
|
|
410
|
+
if (used < limit) return
|
|
411
|
+
account.rejected[dimension] += 1
|
|
412
|
+
throw new QuotaExceededError(scope, dimension, limit, retryAfterMs)
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
private refuseIfOverRolling(account: Account, scope: QuotaScope, dimension: QuotaDimension, window: RollingSum, now: number, limit: number, unit = 1): void {
|
|
416
|
+
if (window.sum(now) < limit) return
|
|
417
|
+
account.rejected[dimension] += 1
|
|
418
|
+
throw new QuotaExceededError(scope, dimension, limit / unit, window.msUntilBelow(now, limit))
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
private maybeSweep(now: number): void {
|
|
422
|
+
this.admissions += 1
|
|
423
|
+
if (this.admissions % SWEEP_EVERY !== 0) return
|
|
424
|
+
for (const [key, account] of this.tenants) {
|
|
425
|
+
if (!account.idle(now)) continue
|
|
426
|
+
this.tenants.delete(key)
|
|
427
|
+
// The tenant's ledger leaves the organization's total with its account; its next turn observes the size again.
|
|
428
|
+
const organizationAccount = this.organizations.get(key.slice(0, key.indexOf('/')))
|
|
429
|
+
if (organizationAccount) organizationAccount.ledgerBytes -= account.ledgerBytes
|
|
430
|
+
}
|
|
431
|
+
for (const [key, account] of this.organizations) if (account.idle(now) && account.ledgerBytes === 0) this.organizations.delete(key)
|
|
432
|
+
}
|
|
433
|
+
}
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The trusted seam of the AaaS line (docs/rfc/aaas-rfc.md §4.4, threats S-1, S-3, S-13).
|
|
3
|
+
*
|
|
4
|
+
* Exactly one place turns "who is calling" into "whose ledger": `deriveSeamTenant`. It takes the caller an
|
|
5
|
+
* authenticator vouched for, and a per-organization key, and returns the ledger tenant as an HMAC digest
|
|
6
|
+
* (`deriveLedgerTenantId`). Nothing a request body, a header, a tool argument or a model output says is ever an
|
|
7
|
+
* input to it, and every API that needs a tenant takes the `SeamTenant` it returns, never a string.
|
|
8
|
+
*
|
|
9
|
+
* What the authenticator is, on the wire, is not decided here: the A3 boundary ADR chooses per-organization
|
|
10
|
+
* credentials or signed assertions. The authenticator is an in-process interface so that decision can land
|
|
11
|
+
* without touching the seam.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import type { IncomingHttpHeaders } from 'node:http'
|
|
15
|
+
import { TenantStateError } from '../tenant-state/errors.ts'
|
|
16
|
+
import { assertOrganizationId, deriveLedgerTenantId } from '../tenant-state/ids.ts'
|
|
17
|
+
|
|
18
|
+
declare const seamBrand: unique symbol
|
|
19
|
+
|
|
20
|
+
const MAX_PRINCIPAL_FIELD_CHARS = 256
|
|
21
|
+
|
|
22
|
+
/** The authenticated caller, as vouched for by the BFF-side authenticator. Never built from a payload. */
|
|
23
|
+
export interface AuthenticatedCaller {
|
|
24
|
+
/** The deploying party (the key owner), not the end user. */
|
|
25
|
+
readonly organization: string
|
|
26
|
+
/** The organization's subject namespace (its identity provider). */
|
|
27
|
+
readonly issuer: string
|
|
28
|
+
/** The end user within that namespace. It becomes a ledger tenant and is never stored as a name. */
|
|
29
|
+
readonly subject: string
|
|
30
|
+
/** An authorization attribute inside the tenant, bound as ADR-23 binds it. */
|
|
31
|
+
readonly scope: string
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** The request as an authenticator sees it: read-only, without the body. */
|
|
35
|
+
export interface CallerRequest {
|
|
36
|
+
readonly method: string
|
|
37
|
+
readonly path: string
|
|
38
|
+
readonly headers: IncomingHttpHeaders
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export interface CallerAuthenticator {
|
|
42
|
+
/** The authenticated caller, or `null` when the request carries no valid credential. A throw is treated as the authenticator being down. */
|
|
43
|
+
authenticate(request: CallerRequest): AuthenticatedCaller | null | Promise<AuthenticatedCaller | null>
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export interface OrganizationKeyring {
|
|
47
|
+
/** The per-organization server-side key (at least 32 bytes), or `null` for an organization this service does not serve. */
|
|
48
|
+
keyFor(organization: string): Uint8Array | null
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** A keyring over a fixed set of keys. Key storage and rotation are deployment concerns and are not designed here. */
|
|
52
|
+
export class StaticOrganizationKeyring implements OrganizationKeyring {
|
|
53
|
+
private readonly keys = new Map<string, Uint8Array>()
|
|
54
|
+
|
|
55
|
+
constructor(keys: Readonly<Record<string, Uint8Array>>) {
|
|
56
|
+
for (const [organization, key] of Object.entries(keys)) {
|
|
57
|
+
assertOrganizationId(organization)
|
|
58
|
+
if (!(key instanceof Uint8Array) || key.byteLength < 32) throw new RangeError(`the key of organization ${organization} must be at least 32 bytes`)
|
|
59
|
+
this.keys.set(organization, Uint8Array.from(key))
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
keyFor(organization: string): Uint8Array | null {
|
|
64
|
+
return this.keys.get(organization) ?? null
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export class SeamError extends TenantStateError {}
|
|
69
|
+
|
|
70
|
+
/** The authenticated principal cannot be turned into a ledger tenant: an empty, oversized or malformed claim. */
|
|
71
|
+
export class InvalidPrincipalError extends SeamError {
|
|
72
|
+
constructor(message: string) {
|
|
73
|
+
super('GOTRY_SEAM_PRINCIPAL_INVALID', message)
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** The caller belongs to an organization this service holds no key for. */
|
|
78
|
+
export class UnknownOrganizationError extends SeamError {
|
|
79
|
+
constructor() {
|
|
80
|
+
super('GOTRY_SEAM_ORGANIZATION_UNKNOWN', 'the organization is not served here')
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Something other than a `SeamTenant` reached an API that needs one. */
|
|
85
|
+
export class SeamTenantRequiredError extends SeamError {
|
|
86
|
+
constructor() {
|
|
87
|
+
super('GOTRY_SEAM_TENANT_REQUIRED', 'a tenant derived at the seam is required')
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* The ledger tenant of one authenticated caller. It can be produced only by `deriveSeamTenant`: the brand is a
|
|
93
|
+
* type-level guard and the registry below is the runtime one, so a look-alike object (parsed JSON, a cast) is refused.
|
|
94
|
+
*/
|
|
95
|
+
export interface SeamTenant {
|
|
96
|
+
readonly organization: string
|
|
97
|
+
readonly ledgerTenant: string
|
|
98
|
+
readonly [seamBrand]: true
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const issued = new WeakSet<object>()
|
|
102
|
+
|
|
103
|
+
export function isSeamTenant(value: unknown): value is SeamTenant {
|
|
104
|
+
return typeof value === 'object' && value !== null && issued.has(value)
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export function assertSeamTenant(value: unknown): SeamTenant {
|
|
108
|
+
if (!isSeamTenant(value)) throw new SeamTenantRequiredError()
|
|
109
|
+
return value
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** `<organization>/<ledger tenant>`: the key for every in-process map and every metric. It contains no personal data. */
|
|
113
|
+
export function seamTenantKey(tenant: SeamTenant): string {
|
|
114
|
+
return `${assertSeamTenant(tenant).organization}/${tenant.ledgerTenant}`
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function principalField(name: string, value: unknown): string {
|
|
118
|
+
if (typeof value !== 'string' || value.length === 0) throw new InvalidPrincipalError(`${name} must be a non-empty string`)
|
|
119
|
+
if (value.length > MAX_PRINCIPAL_FIELD_CHARS) throw new InvalidPrincipalError(`${name} must be at most ${MAX_PRINCIPAL_FIELD_CHARS} characters`)
|
|
120
|
+
return value
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* The only derivation of a ledger tenant from a caller in the service.
|
|
125
|
+
*
|
|
126
|
+
* @throws InvalidTenantIdError for an organization id that is not path-safe, or an issuer or subject that is not well-formed
|
|
127
|
+
* @throws InvalidPrincipalError for a missing or oversized subject, issuer or scope
|
|
128
|
+
* @throws UnknownOrganizationError when the keyring holds no key for the organization
|
|
129
|
+
*/
|
|
130
|
+
export function deriveSeamTenant(caller: AuthenticatedCaller, keyring: OrganizationKeyring): SeamTenant {
|
|
131
|
+
if (typeof caller !== 'object' || caller === null) throw new InvalidPrincipalError('the caller must be an object')
|
|
132
|
+
const organization = assertOrganizationId(caller.organization)
|
|
133
|
+
const issuer = principalField('issuer', caller.issuer)
|
|
134
|
+
const subject = principalField('subject', caller.subject)
|
|
135
|
+
principalField('scope', caller.scope)
|
|
136
|
+
const key = keyring.keyFor(organization)
|
|
137
|
+
if (key === null) throw new UnknownOrganizationError()
|
|
138
|
+
const ledgerTenant = deriveLedgerTenantId(key, issuer, subject)
|
|
139
|
+
const tenant = Object.freeze({ organization, ledgerTenant }) as SeamTenant
|
|
140
|
+
issued.add(tenant)
|
|
141
|
+
return tenant
|
|
142
|
+
}
|