@shiprev/server-sdk-node 0.1.0-beta.0 → 0.1.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -15
- package/dist/auth/token-verifier.d.ts +16 -2
- package/dist/auth/token-verifier.js +34 -7
- package/dist/client/connection-supervisor.d.ts +32 -1
- package/dist/client/connection-supervisor.js +109 -6
- package/dist/client/shiprev-client.d.ts +38 -3
- package/dist/client/shiprev-client.js +146 -6
- package/dist/mirror/degradation.d.ts +22 -0
- package/dist/mirror/degradation.js +67 -0
- package/dist/mirror/entitlement-mirror.d.ts +15 -1
- package/dist/mirror/entitlement-mirror.js +36 -2
- package/dist/ruling/adjudicator.js +5 -12
- package/dist/transport/http.transport.d.ts +6 -1
- package/dist/transport/http.transport.js +43 -2
- package/dist/types.d.ts +106 -2
- package/dist/types.js +48 -1
- package/package.json +10 -11
package/README.md
CHANGED
|
@@ -69,18 +69,19 @@ A runnable version of this flow is the `examples/server-sdk-quickstart` example
|
|
|
69
69
|
|
|
70
70
|
`createClient(options)` and `new ShipRevClient(options)` take the same `ShipRevClientOptions`:
|
|
71
71
|
|
|
72
|
-
| Option | Default | Meaning
|
|
73
|
-
| :--------------------------- | :--------- |
|
|
74
|
-
| `base_url` | (required) | Base URL of the instance's core service; may include a path prefix. Trailing slashes are ignored
|
|
75
|
-
| `api_key` | (required) | SDK-scoped API key of the application
|
|
76
|
-
| `presets.cold_start` | `'deny'` | Ruling for **reads** when the mirror is empty and the instance is unreachable: `'allow'` or `'deny'`
|
|
77
|
-
| `presets.connect_timeout_ms` | `5000` | Timeout of every request to the instance
|
|
78
|
-
| `presets.reservation_ttl_s` | (service) | Default time-to-live of a reservation when `reserve()` does not pass one
|
|
79
|
-
| `buffer.batch_size` | `100` | Usage events sent per request; reaching it triggers a flush
|
|
80
|
-
| `buffer.flush_interval_ms` | `5000` | Background flush interval. `0` disables the timer; the timer never keeps the process alive
|
|
81
|
-
| `backlog` | memory | Where usage waits while it cannot be delivered; see [Persistence modes](#persistence-modes)
|
|
82
|
-
| `on_flush_failure` | | `(events, error) => void`, called when a batch could not be delivered (the events go to the backlog)
|
|
83
|
-
| `on_rejected` | | `(rejected) => void`, called with the events the service refused, each with a `reason`
|
|
72
|
+
| Option | Default | Meaning |
|
|
73
|
+
| :--------------------------- | :--------- | :--------------------------------------------------------------------------------------------------------------------------------------- |
|
|
74
|
+
| `base_url` | (required) | Base URL of the instance's core service; may include a path prefix. Trailing slashes are ignored |
|
|
75
|
+
| `api_key` | (required) | SDK-scoped API key of the application |
|
|
76
|
+
| `presets.cold_start` | `'deny'` | Ruling for **reads** when the mirror is empty and the instance is unreachable: `'allow'` or `'deny'` |
|
|
77
|
+
| `presets.connect_timeout_ms` | `5000` | Timeout of every request to the instance |
|
|
78
|
+
| `presets.reservation_ttl_s` | (service) | Default time-to-live of a reservation when `reserve()` does not pass one |
|
|
79
|
+
| `buffer.batch_size` | `100` | Usage events sent per request; reaching it triggers a flush |
|
|
80
|
+
| `buffer.flush_interval_ms` | `5000` | Background flush interval. `0` disables the timer; the timer never keeps the process alive |
|
|
81
|
+
| `backlog` | memory | Where usage waits while it cannot be delivered; see [Persistence modes](#persistence-modes) |
|
|
82
|
+
| `on_flush_failure` | | `(events, error) => void`, called when a batch could not be delivered (the events go to the backlog) |
|
|
83
|
+
| `on_rejected` | | `(rejected) => void`, called with the events the service refused, each with a `reason` |
|
|
84
|
+
| `on_level_change` | | `(organization_id, level) => void`, called when an organization's effective level changes; see [Degradation levels](#degradation-levels) |
|
|
84
85
|
|
|
85
86
|
## NestJS
|
|
86
87
|
|
|
@@ -163,6 +164,8 @@ export class ReportsController {
|
|
|
163
164
|
|
|
164
165
|
The expected issuer and application come from the service token the client obtains in `start()`; there is nothing to configure. Until the client has obtained one (never reached the instance), every token is refused with `not_ready`. Once known, the expectations are kept while the client is disconnected, and verification keeps working offline for every signing key already fetched. `TokenRejectedError` is exported from the main entry; its message never contains the token.
|
|
165
166
|
|
|
167
|
+
A token whose `kid` is not among the cached signing keys triggers one refetch of the instance's keys, but not within 30 seconds of the last successful fetch: inside that window it is refused (`Unknown key ID`) without any request, so forged tokens with random key IDs cannot multiply your traffic into requests to the instance. A failed fetch does not start the window. After the instance rotates its signing key, tokens signed with the new key may therefore be refused for up to 30 seconds after the previous fetch.
|
|
168
|
+
|
|
166
169
|
## Rulings and `operation`
|
|
167
170
|
|
|
168
171
|
Every ruling names its `operation`: `'read'` or `'write'`. It is required, because the two fail in opposite directions when the instance is unreachable: reads keep working from the last known entitlements, writes stop. Anything that consumes quota, changes state or costs money is a `write`. A missing or unknown `operation`, or a negative or non-finite `quantity`, is refused with `reason: 'invalid_request'`.
|
|
@@ -178,9 +181,9 @@ A `Ruling` is `{ allowed, source, reason?, remaining?, limit?, adjudicated_at }`
|
|
|
178
181
|
| `degraded` | Offline, ruled from the last snapshot |
|
|
179
182
|
| `client_default` | No basis for a ruling: invalid request, cold start offline, or online with nothing in the mirror |
|
|
180
183
|
|
|
181
|
-
Online rulings from the mirror compare `quantity` exactly against the remaining allowance, which the snapshot carries as fixed-point decimal strings. Beyond the remaining allowance
|
|
184
|
+
Online rulings from the mirror compare `quantity` exactly against the remaining allowance, which the snapshot carries as fixed-point decimal strings. Beyond the remaining allowance nothing is allowed locally, whatever the overage policy: whether credits cover the write is state of the instance, so the local answer is a refusal with `reason: 'quota_exhausted'` (`'plan_upgrade_required'` when the capability's overage policy is `PROMPT_UPGRADE`; the instance's ruling uses the same two reasons), which `adjudicateAsync()` turns into a question to the instance and returns its ruling. When no ruling can be had the write stays refused. A capability the mirror does not hold is refused with `reason: 'capability_not_granted'` (which `adjudicateAsync()` turns into a question to the instance).
|
|
182
185
|
|
|
183
|
-
`guardWrite(options, action)` runs `action` only when a `write` ruling allows it
|
|
186
|
+
`guardWrite(options, action)` runs `action` only when a `write` ruling allows it. A refusal with `reason: 'quota_exhausted'` throws `QuotaExhaustedError`, one with `reason: 'plan_upgrade_required'` throws `PlanUpgradeRequiredError`, and any other refusal (disconnected, a capability the mirror does not hold) throws `DisconnectedWriteError`. Both funds errors carry `code` (`'QUOTA_EXHAUSTED'`, `'PLAN_UPGRADE_REQUIRED'`).
|
|
184
187
|
|
|
185
188
|
## Usage reporting
|
|
186
189
|
|
|
@@ -200,7 +203,7 @@ Before sending the buffer, each flush first drains what is waiting in the backlo
|
|
|
200
203
|
|
|
201
204
|
`await flush()` (and `await stop()`) resolves only when every event reported before the call has been accepted, refused (`on_rejected`), or moved to the backlog because it could not be delivered (offline, no service token, or a failed request; `on_flush_failure` is called for a failed buffer batch). If another flush is in flight, for example the one that follows a reconnection, the call waits for it and then makes its own pass. A flush started by `flush()` sends the whole backlog, batch by batch, stopping at the first failed batch; a timer tick skips while a flush is in flight and sends at most one backlog batch.
|
|
202
205
|
|
|
203
|
-
For work whose cost is only known afterwards, reserve first and settle later: `reserve({ organization_id, capability_key, amount, ttl_seconds? })`, then `confirm(reservationId, actualAmount?)` (without an amount, the reserved amount is confirmed) or `release(reservationId)`. Reservations need a service token, which `start()` obtains; without one they throw.
|
|
206
|
+
For work whose cost is only known afterwards, reserve first and settle later: `reserve({ organization_id, capability_key, amount, ttl_seconds? })`, then `confirm(reservationId, actualAmount?)` (without an amount, the reserved amount is confirmed) or `release(reservationId)`. Reservations need a service token, which `start()` obtains; without one they throw. A reservation refused for funds is HTTP `400` with `error.code` `QUOTA_EXHAUSTED` (credits short: offer a top-up) or `PLAN_UPGRADE_REQUIRED` (`PROMPT_UPGRADE` policy: offer a larger plan); `reserve()` throws `QuotaExhaustedError` or `PlanUpgradeRequiredError` accordingly, chosen by the code and never by the message, with `code`, `status` and `request_id`. It is not retried. Any other refusal throws as before.
|
|
204
207
|
|
|
205
208
|
## Connection
|
|
206
209
|
|
|
@@ -228,6 +231,26 @@ Writes are refused whenever the push stream is down, even if other requests to t
|
|
|
228
231
|
|
|
229
232
|
`setOnline(false)` closes the push stream and suspends reconnection until `setOnline(true)`, which sets `LIVE` at once and resumes; on a started client the next connection attempt then corrects the state. Both are for fault injection and tests. `loadSnapshot(snapshot)` seeds the mirror without the network (a `COLD_OFFLINE` client becomes `DISCONNECTED`).
|
|
230
233
|
|
|
234
|
+
## Degradation levels
|
|
235
|
+
|
|
236
|
+
Every organization entry of the snapshot carries two fields the instance computes: `degradation_level`, one of `NONE`, `RESTRICT_NEW`, `RESTRICT_USAGE`, `RESTRICT_READ` and `SUSPENDED` (strictly increasing), and `valid_until`, the time until which the entry may be used without a newer answer. An instance sets the level from its own rules (for example a buyer's unpaid invoices); at `SUSPENDED` the entry's `items` is empty, so nothing is granted. Instances that predate the fields leave them out.
|
|
237
|
+
|
|
238
|
+
`effectiveLevel(organizationId)` combines them with the connection, synchronously and without I/O, into `{ level, source, expired }` (`undefined` when the mirror does not hold the organization):
|
|
239
|
+
|
|
240
|
+
| The client observes | `level` | `source` | `expired` |
|
|
241
|
+
| :----------------------------------------------------- | :----------------------------------------------------- | :----------- | :-------- |
|
|
242
|
+
| `LIVE` (the instance answers) | The entry's level | `'answer'` | `false` |
|
|
243
|
+
| Not `LIVE`, before the entry's `valid_until` | The entry's level | `'degraded'` | `false` |
|
|
244
|
+
| Not `LIVE`, at or after `valid_until` (or none usable) | The entry's level, raised to `RESTRICT_USAGE` at least | `'degraded'` | `true` |
|
|
245
|
+
|
|
246
|
+
A level the client does not know (another string, or not a string) reads as `RESTRICT_USAGE`, never looser. The level does not change `adjudicate()`, `adjudicateAsync()` or `guardWrite()`: a disconnected write is refused whatever the level; what a level blocks in your service is your decision.
|
|
247
|
+
|
|
248
|
+
While `LIVE`, once half of an entry's validity has passed without a newer entry, the client pulls that organization once to renew it (never more than once per entry, never on a fixed period).
|
|
249
|
+
|
|
250
|
+
`refresh(organizationId, { timeout_ms? })` pulls one organization now, sharing a pull already in flight for it, and resolves with the answer's level (`source: 'answer'`); when the pull fails or does not finish within `timeout_ms` (default `presets.connect_timeout_ms`) it resolves with `effectiveLevel()` as it stands. Use it before refusing an action on a level, so that a change at the instance applies within the action.
|
|
251
|
+
|
|
252
|
+
`sendUsage(events)` sends one batch at once, outside the buffer and the backlog, and resolves with `{ accepted_count, duplicates_dropped, rejected }`. Quantities may be decimal strings, sent unchanged. It throws `UsageDeliveryError` when the batch did not get through (network error, timeout, `5xx`); any other refusal throws as it came. Sending the same batch again is safe, as events are deduplicated by `usage_event_id`. It is for callers that keep their own durable queue; everyone else uses `reportUsage()`.
|
|
253
|
+
|
|
231
254
|
## Persistence modes
|
|
232
255
|
|
|
233
256
|
`backlog.mode` chooses where undeliverable usage waits:
|
|
@@ -9,21 +9,35 @@ export interface TokenExpectations {
|
|
|
9
9
|
projectId: string;
|
|
10
10
|
}
|
|
11
11
|
export declare function jwkToPem(jwk: JwksKey): string;
|
|
12
|
+
/**
|
|
13
|
+
* After a successful JWKS fetch, a token with an unknown `kid` does not trigger another fetch for
|
|
14
|
+
* this long; it is refused instead. Without it, an anonymous caller could turn every forged token
|
|
15
|
+
* carrying a random `kid` into one request to the instance. A key rotated in during this window is
|
|
16
|
+
* accepted at most this long after the previous fetch.
|
|
17
|
+
*/
|
|
18
|
+
export declare const JWKS_REFRESH_COOLDOWN_MS = 30000;
|
|
12
19
|
export declare class TokenVerifier {
|
|
13
20
|
private readonly transport;
|
|
14
21
|
private readonly publicKeyCache;
|
|
22
|
+
/** `Date.now()` of the last successful JWKS fetch; failed fetches do not set it. */
|
|
23
|
+
private lastRefreshAt;
|
|
24
|
+
private inFlightRefresh;
|
|
15
25
|
constructor(transport: HttpTransport);
|
|
16
26
|
/**
|
|
17
|
-
* Refreshes the local in-memory public key cache from the JWKS endpoint
|
|
27
|
+
* Refreshes the local in-memory public key cache from the JWKS endpoint. Always fetches (no
|
|
28
|
+
* cooldown); concurrent calls share one request. A success starts the unknown-`kid` cooldown.
|
|
18
29
|
*/
|
|
19
30
|
refreshKeys(): Promise<void>;
|
|
31
|
+
private fetchKeys;
|
|
32
|
+
private inRefreshCooldown;
|
|
20
33
|
/**
|
|
21
34
|
* Directly caches a key in memory (e.g. for testing or preloading)
|
|
22
35
|
*/
|
|
23
36
|
setPublicKey(kid: string, pem: string): void;
|
|
24
37
|
/**
|
|
25
38
|
* Verifies an RS256 JWT token locally, extracting the subject and isolation keys.
|
|
26
|
-
* If the kid is unknown in cache, it triggers a single JWKS refresh
|
|
39
|
+
* If the kid is unknown in cache, it triggers a single JWKS refresh, unless the last successful
|
|
40
|
+
* refresh is less than `JWKS_REFRESH_COOLDOWN_MS` old, in which case the token is refused at once.
|
|
27
41
|
*
|
|
28
42
|
* With `expectations` (or `null`, meaning they are not known yet), the token must also be a buyer
|
|
29
43
|
* session token of that issuer and application; the checks run after the signature, in the order
|
|
@@ -33,7 +33,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.TokenVerifier = void 0;
|
|
36
|
+
exports.TokenVerifier = exports.JWKS_REFRESH_COOLDOWN_MS = void 0;
|
|
37
37
|
exports.jwkToPem = jwkToPem;
|
|
38
38
|
exports.expectationsFromMachineToken = expectationsFromMachineToken;
|
|
39
39
|
exports.expiryOfToken = expiryOfToken;
|
|
@@ -59,21 +59,44 @@ function jwkToPem(jwk) {
|
|
|
59
59
|
});
|
|
60
60
|
return keyObj.export({ type: 'spki', format: 'pem' });
|
|
61
61
|
}
|
|
62
|
+
/**
|
|
63
|
+
* After a successful JWKS fetch, a token with an unknown `kid` does not trigger another fetch for
|
|
64
|
+
* this long; it is refused instead. Without it, an anonymous caller could turn every forged token
|
|
65
|
+
* carrying a random `kid` into one request to the instance. A key rotated in during this window is
|
|
66
|
+
* accepted at most this long after the previous fetch.
|
|
67
|
+
*/
|
|
68
|
+
exports.JWKS_REFRESH_COOLDOWN_MS = 30_000;
|
|
62
69
|
class TokenVerifier {
|
|
63
70
|
transport;
|
|
64
71
|
publicKeyCache = new Map();
|
|
72
|
+
/** `Date.now()` of the last successful JWKS fetch; failed fetches do not set it. */
|
|
73
|
+
lastRefreshAt = null;
|
|
74
|
+
inFlightRefresh = null;
|
|
65
75
|
constructor(transport) {
|
|
66
76
|
this.transport = transport;
|
|
67
77
|
}
|
|
68
78
|
/**
|
|
69
|
-
* Refreshes the local in-memory public key cache from the JWKS endpoint
|
|
79
|
+
* Refreshes the local in-memory public key cache from the JWKS endpoint. Always fetches (no
|
|
80
|
+
* cooldown); concurrent calls share one request. A success starts the unknown-`kid` cooldown.
|
|
70
81
|
*/
|
|
71
82
|
async refreshKeys() {
|
|
83
|
+
if (!this.inFlightRefresh) {
|
|
84
|
+
this.inFlightRefresh = this.fetchKeys().finally(() => {
|
|
85
|
+
this.inFlightRefresh = null;
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
return this.inFlightRefresh;
|
|
89
|
+
}
|
|
90
|
+
async fetchKeys() {
|
|
72
91
|
const jwks = await this.transport.fetchJwks();
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
this.publicKeyCache.set(
|
|
92
|
+
const pems = jwks.keys.map((key) => [key.kid, jwkToPem(key)]);
|
|
93
|
+
for (const [kid, pem] of pems) {
|
|
94
|
+
this.publicKeyCache.set(kid, pem);
|
|
76
95
|
}
|
|
96
|
+
this.lastRefreshAt = Date.now();
|
|
97
|
+
}
|
|
98
|
+
inRefreshCooldown() {
|
|
99
|
+
return this.lastRefreshAt !== null && Date.now() - this.lastRefreshAt < exports.JWKS_REFRESH_COOLDOWN_MS;
|
|
77
100
|
}
|
|
78
101
|
/**
|
|
79
102
|
* Directly caches a key in memory (e.g. for testing or preloading)
|
|
@@ -83,7 +106,8 @@ class TokenVerifier {
|
|
|
83
106
|
}
|
|
84
107
|
/**
|
|
85
108
|
* Verifies an RS256 JWT token locally, extracting the subject and isolation keys.
|
|
86
|
-
* If the kid is unknown in cache, it triggers a single JWKS refresh
|
|
109
|
+
* If the kid is unknown in cache, it triggers a single JWKS refresh, unless the last successful
|
|
110
|
+
* refresh is less than `JWKS_REFRESH_COOLDOWN_MS` old, in which case the token is refused at once.
|
|
87
111
|
*
|
|
88
112
|
* With `expectations` (or `null`, meaning they are not known yet), the token must also be a buyer
|
|
89
113
|
* session token of that issuer and application; the checks run after the signature, in the order
|
|
@@ -116,8 +140,11 @@ class TokenVerifier {
|
|
|
116
140
|
}
|
|
117
141
|
// Lookup public key from local cache
|
|
118
142
|
let publicKeyPem = this.publicKeyCache.get(kid);
|
|
119
|
-
// Cache miss / key rotation: refresh keys once
|
|
143
|
+
// Cache miss / key rotation: refresh keys once, outside the cooldown only
|
|
120
144
|
if (!publicKeyPem) {
|
|
145
|
+
if (this.inRefreshCooldown()) {
|
|
146
|
+
throw new Error(`Unknown key ID: kid=${kid}`);
|
|
147
|
+
}
|
|
121
148
|
await this.refreshKeys();
|
|
122
149
|
publicKeyPem = this.publicKeyCache.get(kid);
|
|
123
150
|
if (!publicKeyPem) {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { EntitlementMirror } from '../mirror/entitlement-mirror';
|
|
2
2
|
import { HttpTransport } from '../transport/http.transport';
|
|
3
|
+
import { OrganizationSnapshot, Snapshot } from '../types';
|
|
3
4
|
/**
|
|
4
5
|
* Keeps a started client connected to its instance, from `start()` to `stop()`:
|
|
5
6
|
*
|
|
@@ -12,7 +13,9 @@ import { HttpTransport } from '../transport/http.transport';
|
|
|
12
13
|
* new one is ready, without pulling; on an unplanned break reconnects once at once, then backs
|
|
13
14
|
* off (1, 2, 4, 8, 16, 30 s, ±20 %);
|
|
14
15
|
* - turns `snapshot.changed` into per-organization refills (at most one in flight per organization)
|
|
15
|
-
* and `snapshot.resync` into a full pull
|
|
16
|
+
* and `snapshot.resync` into a full pull;
|
|
17
|
+
* - while online, renews an organization entry that carries `valid_until` once half of its validity has passed
|
|
18
|
+
* without a newer entry, with one pull of that organization. It never pulls on a fixed period.
|
|
16
19
|
*
|
|
17
20
|
* Every timer is unref'd, so the supervisor never keeps the host process alive.
|
|
18
21
|
*/
|
|
@@ -31,6 +34,18 @@ export interface SupervisorHost {
|
|
|
31
34
|
/** Delivers buffered and backlogged usage. */
|
|
32
35
|
flush(): Promise<void>;
|
|
33
36
|
}
|
|
37
|
+
/**
|
|
38
|
+
* What one per-organization pull did to the mirror: `applied` (the answer's entry is now the mirror's entry),
|
|
39
|
+
* `removed` (the answer no longer holds the organization and the mirror dropped it), or `ignored` (the answer
|
|
40
|
+
* was not used: a newer entry was already held or arrived meanwhile, an unsupported schema, or the client was
|
|
41
|
+
* stopped or paused meanwhile). Only an applied or removed pull is an answer about the organization.
|
|
42
|
+
*/
|
|
43
|
+
export interface OrganizationPull {
|
|
44
|
+
envelope: Snapshot;
|
|
45
|
+
outcome: 'applied' | 'removed' | 'ignored';
|
|
46
|
+
/** The mirror's entry after an applied pull. */
|
|
47
|
+
entry?: OrganizationSnapshot;
|
|
48
|
+
}
|
|
34
49
|
export declare class ConnectionSupervisor {
|
|
35
50
|
private readonly host;
|
|
36
51
|
private token;
|
|
@@ -55,8 +70,13 @@ export declare class ConnectionSupervisor {
|
|
|
55
70
|
private renewTimer;
|
|
56
71
|
private reconnectTimer;
|
|
57
72
|
private watchdogTimer;
|
|
73
|
+
private renewalTimer;
|
|
58
74
|
private readonly refillTargets;
|
|
59
75
|
private readonly refillsInFlight;
|
|
76
|
+
/** Per-organization pulls in flight, shared by refills, renewals and `refresh()`. */
|
|
77
|
+
private readonly organizationPulls;
|
|
78
|
+
/** Organizations whose last renewal pull failed, with the time a renewal may be tried again. */
|
|
79
|
+
private readonly renewalRetryAt;
|
|
60
80
|
constructor(host: SupervisorHost);
|
|
61
81
|
/** The machine token in use, or `null`. */
|
|
62
82
|
currentToken(): string | null;
|
|
@@ -109,6 +129,17 @@ export declare class ConnectionSupervisor {
|
|
|
109
129
|
private onChanged;
|
|
110
130
|
/** Refills one organization until the mirror reaches the highest announced revision. */
|
|
111
131
|
private refill;
|
|
132
|
+
/**
|
|
133
|
+
* One pull of one organization, shared by every concurrent caller, merged into the mirror (or removing the
|
|
134
|
+
* organization when the answer no longer holds it). A pull that finishes after `stop()` or `pause()` leaves
|
|
135
|
+
* the mirror as it is.
|
|
136
|
+
*/
|
|
137
|
+
pullOrganization(organizationId: string): Promise<OrganizationPull>;
|
|
138
|
+
/** Re-arms the renewal timer for the earliest entry due; nothing while offline. Called on every mirror change. */
|
|
139
|
+
scheduleRenewals(): void;
|
|
140
|
+
/** Halfway between receiving an entry and its `valid_until`; undefined for an entry without a usable one. */
|
|
141
|
+
private renewalDue;
|
|
142
|
+
private renewDueEntries;
|
|
112
143
|
/** Full pull: replaces the whole mirror. */
|
|
113
144
|
private pull;
|
|
114
145
|
/** Drops the stream, every timer and any attempt in flight. */
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.ConnectionSupervisor = void 0;
|
|
4
4
|
const token_verifier_1 = require("../auth/token-verifier");
|
|
5
|
+
const degradation_1 = require("../mirror/degradation");
|
|
6
|
+
const entitlement_mirror_1 = require("../mirror/entitlement-mirror");
|
|
5
7
|
const event_stream_1 = require("../transport/event-stream");
|
|
6
8
|
const http_transport_1 = require("../transport/http.transport");
|
|
7
9
|
/** Retry interval while the instance refuses the API key (401 or 403 on the token exchange). */
|
|
@@ -9,6 +11,10 @@ const REFUSED_RETRY_MS = 30_000;
|
|
|
9
11
|
const BACKOFF_STEPS_MS = [1_000, 2_000, 4_000, 8_000, 16_000, 30_000];
|
|
10
12
|
const RENEW_AT = 0.8;
|
|
11
13
|
const HEARTBEAT_TIMEOUT_FACTOR = 3;
|
|
14
|
+
/** Shortest time between two renewals of the same entry. */
|
|
15
|
+
const MIN_RENEWAL_SPACING_MS = 10_000;
|
|
16
|
+
/** Wait before renewing an entry again after a renewal pull failed. */
|
|
17
|
+
const RENEWAL_RETRY_MS = 30_000;
|
|
12
18
|
function backoffDelay(step) {
|
|
13
19
|
const base = BACKOFF_STEPS_MS[Math.min(step, BACKOFF_STEPS_MS.length - 1)];
|
|
14
20
|
return Math.round(base * (0.8 + Math.random() * 0.4));
|
|
@@ -42,8 +48,13 @@ class ConnectionSupervisor {
|
|
|
42
48
|
renewTimer = null;
|
|
43
49
|
reconnectTimer = null;
|
|
44
50
|
watchdogTimer = null;
|
|
51
|
+
renewalTimer = null;
|
|
45
52
|
refillTargets = new Map();
|
|
46
53
|
refillsInFlight = new Set();
|
|
54
|
+
/** Per-organization pulls in flight, shared by refills, renewals and `refresh()`. */
|
|
55
|
+
organizationPulls = new Map();
|
|
56
|
+
/** Organizations whose last renewal pull failed, with the time a renewal may be tried again. */
|
|
57
|
+
renewalRetryAt = new Map();
|
|
47
58
|
constructor(host) {
|
|
48
59
|
this.host = host;
|
|
49
60
|
}
|
|
@@ -218,6 +229,7 @@ class ConnectionSupervisor {
|
|
|
218
229
|
this.backoffStep = 0;
|
|
219
230
|
this.host.setOnline(true);
|
|
220
231
|
this.armWatchdog();
|
|
232
|
+
this.scheduleRenewals();
|
|
221
233
|
if (flush) {
|
|
222
234
|
void this.host.flush().catch(() => undefined);
|
|
223
235
|
}
|
|
@@ -229,6 +241,7 @@ class ConnectionSupervisor {
|
|
|
229
241
|
this.stream = null;
|
|
230
242
|
}
|
|
231
243
|
this.clearTimer('watchdogTimer');
|
|
244
|
+
this.clearTimer('renewalTimer');
|
|
232
245
|
this.host.setOnline(false);
|
|
233
246
|
}
|
|
234
247
|
/** An unplanned break of the active stream: one immediate reconnect, still LIVE meanwhile. */
|
|
@@ -487,17 +500,17 @@ class ConnectionSupervisor {
|
|
|
487
500
|
this.refillsInFlight.add(organizationId);
|
|
488
501
|
try {
|
|
489
502
|
for (;;) {
|
|
490
|
-
const envelope = await this.
|
|
503
|
+
const { envelope, outcome } = await this.pullOrganization(organizationId);
|
|
491
504
|
if (!this.running)
|
|
492
505
|
return;
|
|
493
|
-
if (envelope.organizations
|
|
494
|
-
this.host.mirror.mergeSnapshot(envelope);
|
|
495
|
-
}
|
|
496
|
-
else {
|
|
497
|
-
this.host.mirror.removeOrganization(organizationId);
|
|
506
|
+
if (!envelope.organizations || !(organizationId in envelope.organizations)) {
|
|
498
507
|
this.refillTargets.delete(organizationId);
|
|
499
508
|
return;
|
|
500
509
|
}
|
|
510
|
+
// An answer the mirror did not take (stale or unsupported) is not asked again at once; the target
|
|
511
|
+
// stays for the next notification or full pull.
|
|
512
|
+
if (outcome === 'ignored')
|
|
513
|
+
return;
|
|
501
514
|
const target = this.refillTargets.get(organizationId) ?? 0;
|
|
502
515
|
if ((this.host.mirror.getOrganizationRevision(organizationId) ?? 0) >= target) {
|
|
503
516
|
this.refillTargets.delete(organizationId);
|
|
@@ -512,6 +525,95 @@ class ConnectionSupervisor {
|
|
|
512
525
|
this.refillsInFlight.delete(organizationId);
|
|
513
526
|
}
|
|
514
527
|
}
|
|
528
|
+
/**
|
|
529
|
+
* One pull of one organization, shared by every concurrent caller, merged into the mirror (or removing the
|
|
530
|
+
* organization when the answer no longer holds it). A pull that finishes after `stop()` or `pause()` leaves
|
|
531
|
+
* the mirror as it is.
|
|
532
|
+
*/
|
|
533
|
+
pullOrganization(organizationId) {
|
|
534
|
+
const inFlight = this.organizationPulls.get(organizationId);
|
|
535
|
+
if (inFlight)
|
|
536
|
+
return inFlight;
|
|
537
|
+
const generation = this.generation;
|
|
538
|
+
const pull = (async () => {
|
|
539
|
+
const heldBefore = this.host.mirror.getOrganization(organizationId);
|
|
540
|
+
const envelope = await this.authorized((token) => this.host.transport.fetchSnapshot(token, organizationId));
|
|
541
|
+
const usable = generation === this.generation &&
|
|
542
|
+
typeof envelope?.schema_version === 'number' &&
|
|
543
|
+
envelope.schema_version <= entitlement_mirror_1.MAX_SUPPORTED_SCHEMA_VERSION;
|
|
544
|
+
if (!usable)
|
|
545
|
+
return { envelope, outcome: 'ignored' };
|
|
546
|
+
const answered = envelope.organizations?.[organizationId];
|
|
547
|
+
if (answered) {
|
|
548
|
+
this.host.mirror.mergeSnapshot(envelope);
|
|
549
|
+
const held = this.host.mirror.getOrganization(organizationId);
|
|
550
|
+
return held === answered ? { envelope, outcome: 'applied', entry: held } : { envelope, outcome: 'ignored' };
|
|
551
|
+
}
|
|
552
|
+
// Drop the organization only if the mirror still holds what it held when the pull started: a newer entry
|
|
553
|
+
// that arrived meanwhile is not removed by an older answer.
|
|
554
|
+
if (this.host.mirror.getOrganization(organizationId) !== heldBefore)
|
|
555
|
+
return { envelope, outcome: 'ignored' };
|
|
556
|
+
this.host.mirror.removeOrganization(organizationId);
|
|
557
|
+
return { envelope, outcome: 'removed' };
|
|
558
|
+
})();
|
|
559
|
+
this.organizationPulls.set(organizationId, pull);
|
|
560
|
+
const clear = () => {
|
|
561
|
+
if (this.organizationPulls.get(organizationId) === pull)
|
|
562
|
+
this.organizationPulls.delete(organizationId);
|
|
563
|
+
};
|
|
564
|
+
void pull.then(clear, clear);
|
|
565
|
+
return pull;
|
|
566
|
+
}
|
|
567
|
+
// ─── Renewal of entries with a validity ───────────────────────────────────
|
|
568
|
+
/** Re-arms the renewal timer for the earliest entry due; nothing while offline. Called on every mirror change. */
|
|
569
|
+
scheduleRenewals() {
|
|
570
|
+
this.clearTimer('renewalTimer');
|
|
571
|
+
if (!this.running || this.paused || !this.live)
|
|
572
|
+
return;
|
|
573
|
+
let next = Number.POSITIVE_INFINITY;
|
|
574
|
+
for (const organizationId of this.host.mirror.organizationIds()) {
|
|
575
|
+
const due = this.renewalDue(organizationId);
|
|
576
|
+
if (due !== undefined)
|
|
577
|
+
next = Math.min(next, due);
|
|
578
|
+
}
|
|
579
|
+
if (next === Number.POSITIVE_INFINITY)
|
|
580
|
+
return;
|
|
581
|
+
this.renewalTimer = unrefTimer(() => {
|
|
582
|
+
this.renewalTimer = null;
|
|
583
|
+
this.renewDueEntries();
|
|
584
|
+
}, next - Date.now());
|
|
585
|
+
}
|
|
586
|
+
/** Halfway between receiving an entry and its `valid_until`; undefined for an entry without a usable one. */
|
|
587
|
+
renewalDue(organizationId) {
|
|
588
|
+
const entry = this.host.mirror.getOrganization(organizationId);
|
|
589
|
+
const receivedAt = this.host.mirror.getReceivedAt(organizationId);
|
|
590
|
+
if (!entry || receivedAt === undefined)
|
|
591
|
+
return undefined;
|
|
592
|
+
const validUntil = (0, degradation_1.validUntilOf)(entry);
|
|
593
|
+
if (validUntil === undefined || validUntil <= receivedAt)
|
|
594
|
+
return undefined;
|
|
595
|
+
const due = Math.max(receivedAt + (validUntil - receivedAt) / 2, receivedAt + MIN_RENEWAL_SPACING_MS);
|
|
596
|
+
return Math.max(due, this.renewalRetryAt.get(organizationId) ?? 0);
|
|
597
|
+
}
|
|
598
|
+
renewDueEntries() {
|
|
599
|
+
if (!this.running || this.paused || !this.live)
|
|
600
|
+
return;
|
|
601
|
+
const now = Date.now();
|
|
602
|
+
for (const organizationId of this.host.mirror.organizationIds()) {
|
|
603
|
+
const due = this.renewalDue(organizationId);
|
|
604
|
+
if (due === undefined || due > now)
|
|
605
|
+
continue;
|
|
606
|
+
this.renewalRetryAt.set(organizationId, now + RENEWAL_RETRY_MS);
|
|
607
|
+
// The retry time set above stays unless the renewal was taken: a stale or unsupported answer does not
|
|
608
|
+
// move the entry's receipt time, and renewing it again at once would pull without end.
|
|
609
|
+
void this.pullOrganization(organizationId).then(({ outcome }) => {
|
|
610
|
+
if (outcome !== 'ignored')
|
|
611
|
+
this.renewalRetryAt.delete(organizationId);
|
|
612
|
+
this.scheduleRenewals();
|
|
613
|
+
}, () => this.scheduleRenewals());
|
|
614
|
+
}
|
|
615
|
+
this.scheduleRenewals();
|
|
616
|
+
}
|
|
515
617
|
/** Full pull: replaces the whole mirror. */
|
|
516
618
|
async pull() {
|
|
517
619
|
const snapshot = await this.authorized((token) => this.host.transport.fetchSnapshot(token));
|
|
@@ -535,6 +637,7 @@ class ConnectionSupervisor {
|
|
|
535
637
|
this.clearTimer('renewTimer');
|
|
536
638
|
this.clearTimer('reconnectTimer');
|
|
537
639
|
this.clearTimer('watchdogTimer');
|
|
640
|
+
this.clearTimer('renewalTimer');
|
|
538
641
|
}
|
|
539
642
|
clearTimer(name) {
|
|
540
643
|
const timer = this[name];
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { UsageBuffer } from '../metering/usage-buffer';
|
|
2
2
|
import { HttpTransport } from '../transport/http.transport';
|
|
3
|
-
import { AdjudicateOptions, ClientState, ConfirmReservationResult, ReleaseReservationResult, ReservationResult, ReserveOptions, Ruling, ShipRevClientOptions, Snapshot, TokenClaims, UsageEventItem } from '../types';
|
|
3
|
+
import { AdjudicateOptions, ClientState, ConfirmReservationResult, EffectiveLevel, ReleaseReservationResult, ReservationResult, ReserveOptions, Ruling, ShipRevClientOptions, Snapshot, TokenClaims, UsageEventItem, UsageSendEvent, UsageSendResult } from '../types';
|
|
4
4
|
export declare class ShipRevClient {
|
|
5
5
|
private readonly options;
|
|
6
6
|
private readonly transport;
|
|
@@ -16,6 +16,10 @@ export declare class ShipRevClient {
|
|
|
16
16
|
private expectations;
|
|
17
17
|
/** Amounts reserved by this client, so `confirm()` without an amount confirms what was reserved. */
|
|
18
18
|
private readonly reservedAmounts;
|
|
19
|
+
/** Effective levels last reported through `on_level_change`. */
|
|
20
|
+
private readonly reportedLevels;
|
|
21
|
+
/** Fires when the earliest `valid_until` passes while disconnected, to report the expiry. */
|
|
22
|
+
private expiryTimer;
|
|
19
23
|
constructor(options: ShipRevClientOptions);
|
|
20
24
|
/**
|
|
21
25
|
* Connects the client: fetches the signing keys, exchanges the API key for a service token, opens
|
|
@@ -55,8 +59,34 @@ export declare class ShipRevClient {
|
|
|
55
59
|
*/
|
|
56
60
|
adjudicateAsync(options: AdjudicateOptions): Promise<Ruling>;
|
|
57
61
|
/**
|
|
58
|
-
*
|
|
59
|
-
*
|
|
62
|
+
* The effective degradation level of an organization: the level of the mirror entry while the instance
|
|
63
|
+
* answers; without an answer, the entry's level until its `valid_until`, then at least `RESTRICT_USAGE`
|
|
64
|
+
* (marked `expired`). `undefined` when the mirror does not hold the organization. Synchronous, no I/O.
|
|
65
|
+
*
|
|
66
|
+
* It does not change `adjudicate()` or `guardWrite()`: a disconnected write is refused whatever the level.
|
|
67
|
+
*/
|
|
68
|
+
effectiveLevel(organizationId: string): EffectiveLevel | undefined;
|
|
69
|
+
/**
|
|
70
|
+
* Pulls one organization now (sharing a pull already in flight for it) and resolves with the level of the
|
|
71
|
+
* answer, `source: 'answer'`. When the pull fails or does not finish within `timeout_ms` (default: the
|
|
72
|
+
* connect timeout), resolves with `effectiveLevel()` as it stands. `undefined` when the instance no longer
|
|
73
|
+
* holds the organization, or the mirror never did.
|
|
74
|
+
*/
|
|
75
|
+
refresh(organizationId: string, options?: {
|
|
76
|
+
timeout_ms?: number;
|
|
77
|
+
}): Promise<EffectiveLevel | undefined>;
|
|
78
|
+
/**
|
|
79
|
+
* Sends a batch of usage events at once, outside the buffer and the backlog, and resolves with what the
|
|
80
|
+
* instance did with each. A 401 renews the token once, as every request does. Throws `UsageDeliveryError`
|
|
81
|
+
* when the batch did not reach the instance or it failed to process it (network error, timeout, 5xx); any
|
|
82
|
+
* other refusal throws as it came. Sending the same batch again is safe: events are deduplicated by key.
|
|
83
|
+
*/
|
|
84
|
+
sendUsage(events: UsageSendEvent[]): Promise<UsageSendResult>;
|
|
85
|
+
/**
|
|
86
|
+
* Guard for write operations (fail-closed): runs `action` only when the local ruling allows the write.
|
|
87
|
+
* A refusal for funds throws `QuotaExhaustedError` (`quota_exhausted`) or `PlanUpgradeRequiredError`
|
|
88
|
+
* (`plan_upgrade_required`, PROMPT_UPGRADE); any other refusal, a disconnected write included, throws
|
|
89
|
+
* `DisconnectedWriteError` as before.
|
|
60
90
|
*/
|
|
61
91
|
guardWrite<T>(options: Omit<AdjudicateOptions, 'operation'>, action: () => Promise<T> | T): Promise<T>;
|
|
62
92
|
/**
|
|
@@ -101,6 +131,11 @@ export declare class ShipRevClient {
|
|
|
101
131
|
getUsageBuffer(): UsageBuffer;
|
|
102
132
|
getTransport(): HttpTransport;
|
|
103
133
|
private applyOnline;
|
|
134
|
+
private onMirrorChange;
|
|
135
|
+
/** Reports every organization whose effective level differs from what was last reported. */
|
|
136
|
+
private notifyLevels;
|
|
137
|
+
/** While disconnected, wakes up when the earliest entry still valid expires, to report it. */
|
|
138
|
+
private armExpiryTimer;
|
|
104
139
|
private learnExpectations;
|
|
105
140
|
private refillOrganization;
|
|
106
141
|
}
|