@ultimat3/core 29.3.2 → 30.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +45 -81
- package/README.md +137 -10
- package/package.json +19 -3
- package/src/address-class.ts +53 -12
- package/src/aws-sigv4.ts +104 -8
- package/src/breaker.ts +177 -0
- package/src/client-dispatch.ts +76 -17
- package/src/client-flight.ts +4 -4
- package/src/client-paths-pins.ts +92 -0
- package/src/client-problem.ts +25 -1
- package/src/client-scope.ts +3 -1
- package/src/client-stream.ts +106 -0
- package/src/client-transport.ts +16 -2
- package/src/config-assets.ts +64 -0
- package/src/config-client-errors.ts +132 -0
- package/src/config-defaults.ts +18 -4
- package/src/config-failure-capture.ts +70 -0
- package/src/config-health.ts +8 -0
- package/src/config-islands.ts +1 -1
- package/src/config-keys.ts +2 -0
- package/src/config-site.ts +16 -0
- package/src/config-time.ts +40 -0
- package/src/config.ts +37 -26
- package/src/context.ts +92 -10
- package/src/core-error-codes.ts +19 -0
- package/src/cursor.ts +7 -3
- package/src/deadline-signal.ts +57 -0
- package/src/declared-errors-openapi.ts +70 -0
- package/src/declared-errors-scan.ts +79 -0
- package/src/declared-errors.ts +185 -0
- package/src/define-error.ts +200 -0
- package/src/deprecation.ts +14 -3
- package/src/drain-deadline.ts +22 -0
- package/src/env.ts +8 -3
- package/src/error-fingerprint.ts +95 -0
- package/src/error-reporter-chain.ts +45 -0
- package/src/error-reporter-sentry.ts +4 -0
- package/src/error-reporter.ts +14 -1
- package/src/error-status-declared.ts +82 -0
- package/src/exercise-errors.ts +41 -0
- package/src/exercise-run.ts +176 -0
- package/src/exports/error-contract.ts +7 -0
- package/src/exports/observability.ts +26 -0
- package/src/failure-capture-flush.ts +93 -0
- package/src/failure-capture-mask.ts +54 -0
- package/src/failure-capture.ts +205 -0
- package/src/fill-wire.ts +30 -0
- package/src/finite-option.ts +7 -0
- package/src/health-disclosure.ts +2 -2
- package/src/hmac.ts +47 -0
- package/src/index.ts +172 -9
- package/src/json-storable.ts +21 -0
- package/src/keyed-fingerprint.ts +3 -7
- package/src/lifecycle.ts +5 -1
- package/src/logger.ts +1 -1
- package/src/otlp-metric-exporter.ts +7 -1
- package/src/otlp-post.ts +147 -0
- package/src/otlp-span-exporter.ts +7 -1
- package/src/otlp.ts +4 -37
- package/src/outbound-attempt.ts +235 -0
- package/src/outbound-dial.ts +103 -0
- package/src/outbound-errors.ts +193 -0
- package/src/outbound-hops.ts +135 -0
- package/src/outbound-read.ts +103 -0
- package/src/outbound-subpath.ts +26 -0
- package/src/outbound-types.ts +108 -0
- package/src/outbound.ts +223 -0
- package/src/page-meta.ts +8 -0
- package/src/page.ts +9 -0
- package/src/record-envelope-openapi.ts +6 -0
- package/src/record-envelope.ts +33 -9
- package/src/record-sink.ts +13 -0
- package/src/redis-client.ts +78 -0
- package/src/registrar.ts +11 -0
- package/src/render-node.ts +13 -0
- package/src/render-nodes.ts +137 -0
- package/src/retry-loop.ts +197 -0
- package/src/retry.ts +47 -145
- package/src/roles.ts +2 -2
- package/src/route-path-merge.ts +55 -0
- package/src/route-path-template.ts +124 -0
- package/src/runtime-metrics.ts +16 -1
- package/src/same-origin.ts +18 -2
- package/src/session-lost.ts +115 -0
- package/src/single-flight.ts +75 -40
- package/src/source-mask.ts +36 -2
- package/src/tool-description.ts +29 -0
- package/src/visible-interval.ts +153 -0
- package/src/webhook-signature.ts +9 -11
package/package.json
CHANGED
|
@@ -1,7 +1,18 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ultimat3/core",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "30.0.0",
|
|
4
4
|
"description": "Ultimate's foundation: errors, context, env, config, clock, ids, logging, telemetry, lifecycle",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"ultimate",
|
|
7
|
+
"bun",
|
|
8
|
+
"framework",
|
|
9
|
+
"errors",
|
|
10
|
+
"env",
|
|
11
|
+
"config",
|
|
12
|
+
"telemetry",
|
|
13
|
+
"retry"
|
|
14
|
+
],
|
|
15
|
+
"homepage": "https://github.com/developerz-ai/ultimate/wiki/Configuration",
|
|
5
16
|
"license": "MIT",
|
|
6
17
|
"type": "module",
|
|
7
18
|
"sideEffects": [
|
|
@@ -19,7 +30,12 @@
|
|
|
19
30
|
},
|
|
20
31
|
"exports": {
|
|
21
32
|
".": "./src/index.ts",
|
|
22
|
-
"./page": "./src/page.ts"
|
|
33
|
+
"./page": "./src/page.ts",
|
|
34
|
+
"./internal/error-status": "./src/error-status-declared.ts",
|
|
35
|
+
"./outbound": "./src/outbound-subpath.ts",
|
|
36
|
+
"./redis": "./src/redis-client.ts",
|
|
37
|
+
"./render-node": "./src/render-node.ts",
|
|
38
|
+
"./render-nodes": "./src/render-nodes.ts"
|
|
23
39
|
},
|
|
24
40
|
"files": [
|
|
25
41
|
"src",
|
|
@@ -37,6 +53,6 @@
|
|
|
37
53
|
"test": "bun test"
|
|
38
54
|
},
|
|
39
55
|
"dependencies": {
|
|
40
|
-
"@ultimat3/schema": "
|
|
56
|
+
"@ultimat3/schema": "30.0.0"
|
|
41
57
|
}
|
|
42
58
|
}
|
package/src/address-class.ts
CHANGED
|
@@ -29,6 +29,7 @@ const V4_RULES: readonly (readonly [number, number, AddressClass])[] = [
|
|
|
29
29
|
[0x64400000, 10, 'cgnat'],
|
|
30
30
|
[0xc0000000, 24, 'reserved'], // 192.0.0.0/24, IETF protocol assignments
|
|
31
31
|
[0xc0000200, 24, 'reserved'], // 192.0.2.0/24, TEST-NET-1
|
|
32
|
+
[0xc0586300, 24, 'reserved'], // 192.88.99.0/24, the deprecated 6to4 relay anycast
|
|
32
33
|
[0xc6120000, 15, 'reserved'], // 198.18.0.0/15, benchmarking
|
|
33
34
|
[0xc6336400, 24, 'reserved'], // 198.51.100.0/24, TEST-NET-2
|
|
34
35
|
[0xcb007100, 24, 'reserved'], // 203.0.113.0/24, TEST-NET-3
|
|
@@ -97,24 +98,48 @@ function parseV6(text: string): readonly number[] | undefined {
|
|
|
97
98
|
|
|
98
99
|
const embeddedV4 = (g: readonly number[]): number => (g[6] ?? 0) * 0x10000 + (g[7] ?? 0);
|
|
99
100
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
101
|
+
const zeroUpTo = (g: readonly number[], n: number): boolean =>
|
|
102
|
+
g.slice(0, n).every((group) => group === 0);
|
|
103
|
+
|
|
104
|
+
/** `::ffff:a.b.c.d`: how a dual-stack socket spells an IPv4 peer, so that peer really is the IPv4. */
|
|
105
|
+
const isMappedV4 = (g: readonly number[]): boolean => zeroUpTo(g, 5) && g[5] === 0xffff;
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* The IPv4 a TRANSIT form carries: IPv4-translated `::ffff:0:a.b.c.d`, IPv4-compatible `::a.b.c.d`,
|
|
109
|
+
* NAT64 `64:ff9b::/96` and 6to4 `2002::/16`, which alone holds it in bits 16-47, not the last 32.
|
|
110
|
+
* Each is routed to that address, but the address on the packet is not that host's.
|
|
111
|
+
*/
|
|
112
|
+
function transitV4(g: readonly number[]): number | undefined {
|
|
113
|
+
if (zeroUpTo(g, 4) && g[4] === 0xffff && g[5] === 0) return embeddedV4(g);
|
|
114
|
+
const isLoopbackOrUnspecified = zeroUpTo(g, 7) && (g[7] ?? 0) <= 1;
|
|
115
|
+
if (zeroUpTo(g, 6) && !isLoopbackOrUnspecified) return embeddedV4(g);
|
|
108
116
|
if (g[0] === 0x64 && g[1] === 0xff9b && g.slice(2, 6).every((group) => group === 0)) {
|
|
109
|
-
return
|
|
117
|
+
return embeddedV4(g);
|
|
110
118
|
}
|
|
119
|
+
if (g[0] === 0x2002) return (g[1] ?? 0) * 0x10000 + (g[2] ?? 0);
|
|
120
|
+
return undefined;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function classifyV6(g: readonly number[]): AddressClass {
|
|
124
|
+
if (zeroUpTo(g, 8)) return 'unspecified';
|
|
125
|
+
if (zeroUpTo(g, 7) && g[7] === 1) return 'loopback';
|
|
126
|
+
// Each of these is routed to the IPv4 address it carries, so that is what gets classified.
|
|
127
|
+
if (isMappedV4(g)) return classifyV4(embeddedV4(g));
|
|
128
|
+
const carried = transitV4(g);
|
|
129
|
+
if (carried !== undefined) return classifyV4(carried);
|
|
111
130
|
const first = g[0] ?? 0;
|
|
131
|
+
const second = g[1] ?? 0;
|
|
112
132
|
if ((first & 0xffc0) === 0xfe80) return 'link-local';
|
|
113
133
|
if ((first & 0xffc0) === 0xfec0) return 'private'; // deprecated site-local
|
|
114
134
|
if ((first & 0xfe00) === 0xfc00) return 'ula';
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
if (first
|
|
135
|
+
// Only `2000::/3` is global unicast. The rest — multicast, `100::/64`, local-use NAT64, SRv6, the
|
|
136
|
+
// unassigned space — is named rather than left to read as public, which is what fails closed.
|
|
137
|
+
if ((first & 0xe000) !== 0x2000) return 'reserved';
|
|
138
|
+
// Teredo. Its client address is obfuscated, so unwrapping it would classify a mask, not a host.
|
|
139
|
+
if (first === 0x2001 && second === 0) return 'private';
|
|
140
|
+
if (first === 0x2001 && second < 0x0200) return 'reserved'; // 2001::/23, IETF protocol assignments
|
|
141
|
+
if (first === 0x2001 && second === 0x0db8) return 'reserved'; // documentation
|
|
142
|
+
if (first === 0x3fff && second < 0x1000) return 'reserved'; // 3fff::/20, documentation
|
|
118
143
|
|
|
119
144
|
return 'public';
|
|
120
145
|
}
|
|
@@ -143,6 +168,22 @@ export function classifyAddress(address: string): AddressClass | undefined {
|
|
|
143
168
|
return v4 === undefined ? undefined : classifyV4(v4);
|
|
144
169
|
}
|
|
145
170
|
|
|
171
|
+
/**
|
|
172
|
+
* The class of an address as a CALLER or a local name, for a check that grants something on it —
|
|
173
|
+
* `classifyAddress` answers where a DIAL would land, a different question over the same bits. A
|
|
174
|
+
* transit form spelling an inward IPv4 (`64:ff9b::7f00:1`, `2002:7f00:1::`) is a stranger across
|
|
175
|
+
* the network, so it answers `reserved`, never the class it spells; `::ffff:127.0.0.1` stays
|
|
176
|
+
* `loopback`, being only how a dual-stack socket writes 127.0.0.1. `undefined` as there.
|
|
177
|
+
*/
|
|
178
|
+
export function peerAddressClass(address: string): AddressClass | undefined {
|
|
179
|
+
const kind = classifyAddress(address);
|
|
180
|
+
if (kind === undefined || kind === 'public') return kind;
|
|
181
|
+
const text = literalOf(address);
|
|
182
|
+
if (!text.includes(':')) return kind;
|
|
183
|
+
const groups = parseV6(text);
|
|
184
|
+
return groups !== undefined && transitV4(groups) !== undefined ? 'reserved' : kind;
|
|
185
|
+
}
|
|
186
|
+
|
|
146
187
|
/** Fails CLOSED: anything but a literal classified `public` — a hostname included — is `false`. */
|
|
147
188
|
export function isPublicAddress(address: string): boolean {
|
|
148
189
|
return classifyAddress(address) === 'public';
|
package/src/aws-sigv4.ts
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import { assert } from './assert';
|
|
7
7
|
import { type Clock, systemClock } from './clock';
|
|
8
8
|
import { describeValue } from './error-render';
|
|
9
|
+
import { finiteCount } from './finite-option';
|
|
9
10
|
|
|
10
11
|
export interface AwsCredentials {
|
|
11
12
|
readonly accessKeyId: string;
|
|
@@ -62,7 +63,32 @@ export interface SignedAwsRequest {
|
|
|
62
63
|
readonly signature: string;
|
|
63
64
|
}
|
|
64
65
|
|
|
66
|
+
/**
|
|
67
|
+
* A URL whose query carries the signature: a credential-less holder sends it from a browser. What
|
|
68
|
+
* `headers` names is inside the signature, so the request must carry exactly those values.
|
|
69
|
+
*/
|
|
70
|
+
export interface PresignAwsUrlInput
|
|
71
|
+
extends Pick<
|
|
72
|
+
SignAwsRequestInput,
|
|
73
|
+
'method' | 'url' | 'credentials' | 'region' | 'service' | 'clock' | 'doubleEncodePath'
|
|
74
|
+
> {
|
|
75
|
+
/** 1 … 604 800, S3's own limit. */
|
|
76
|
+
readonly expiresInSeconds: number;
|
|
77
|
+
/** Signed, and sent by whoever holds the URL. `host` is always signed. */
|
|
78
|
+
readonly headers?: Readonly<Record<string, string>> | undefined;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export interface PresignedAwsUrl {
|
|
82
|
+
readonly url: string;
|
|
83
|
+
/** The two intermediate strings, exposed so a provider's `SignatureDoesNotMatch` can be diffed. */
|
|
84
|
+
readonly canonicalRequest: string;
|
|
85
|
+
readonly stringToSign: string;
|
|
86
|
+
readonly signature: string;
|
|
87
|
+
}
|
|
88
|
+
|
|
65
89
|
const ALGORITHM = 'AWS4-HMAC-SHA256';
|
|
90
|
+
/** SigV4's own ceiling on `X-Amz-Expires`: seven days. A presigner beside this one refuses past it too. */
|
|
91
|
+
export const MAX_AWS_PRESIGN_SECONDS = 604_800;
|
|
66
92
|
const SCOPE_SEGMENT = /^[a-z0-9-]{1,64}$/;
|
|
67
93
|
const METHOD = /^[A-Z]{1,16}$/;
|
|
68
94
|
/** Headers the signer writes itself; a caller's copy would sign a value the request does not carry. */
|
|
@@ -175,7 +201,9 @@ function parseUrl(url: string | URL): URL {
|
|
|
175
201
|
return new URL(url);
|
|
176
202
|
}
|
|
177
203
|
|
|
178
|
-
function assertInput(
|
|
204
|
+
function assertInput(
|
|
205
|
+
input: Pick<SignAwsRequestInput, 'credentials' | 'region' | 'service' | 'method'>,
|
|
206
|
+
): void {
|
|
179
207
|
const { accessKeyId, secretAccessKey } = input.credentials;
|
|
180
208
|
assert(
|
|
181
209
|
accessKeyId !== '' && secretAccessKey !== '',
|
|
@@ -214,6 +242,25 @@ const amzDate = (clock: Clock): string =>
|
|
|
214
242
|
.replace(/\.\d{3}Z$/, 'Z')
|
|
215
243
|
.replace(/[-:]/g, '');
|
|
216
244
|
|
|
245
|
+
/** The scope, the string to sign and the signature over it: one derivation for both forms. */
|
|
246
|
+
async function signatureOver(
|
|
247
|
+
input: Pick<SignAwsRequestInput, 'credentials' | 'region' | 'service'>,
|
|
248
|
+
date: string,
|
|
249
|
+
canonicalRequest: string,
|
|
250
|
+
): Promise<{ readonly stringToSign: string; readonly signature: string }> {
|
|
251
|
+
const day = date.slice(0, 8);
|
|
252
|
+
const scope = scopeOf(input, day);
|
|
253
|
+
const stringToSign = [ALGORITHM, date, scope, await sha256Hex(canonicalRequest)].join('\n');
|
|
254
|
+
let key = encoder.encode(`AWS4${input.credentials.secretAccessKey}`);
|
|
255
|
+
for (const part of [day, input.region, input.service, 'aws4_request']) {
|
|
256
|
+
key = await hmac(key, part);
|
|
257
|
+
}
|
|
258
|
+
return { stringToSign, signature: toHex(await hmac(key, stringToSign)) };
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
const scopeOf = (input: Pick<SignAwsRequestInput, 'region' | 'service'>, day: string): string =>
|
|
262
|
+
`${day}/${input.region}/${input.service}/aws4_request`;
|
|
263
|
+
|
|
217
264
|
/**
|
|
218
265
|
* Sign one request with SigV4 in the `Authorization` header. Returns everything to send — the
|
|
219
266
|
* re-encoded URL and every header — plus the canonical request and string-to-sign for debugging.
|
|
@@ -253,13 +300,8 @@ export async function signAwsRequest(input: SignAwsRequestInput): Promise<Signed
|
|
|
253
300
|
hash,
|
|
254
301
|
].join('\n');
|
|
255
302
|
|
|
256
|
-
const
|
|
257
|
-
const scope =
|
|
258
|
-
const stringToSign = [ALGORITHM, date, scope, await sha256Hex(canonicalRequest)].join('\n');
|
|
259
|
-
let key = encoder.encode(`AWS4${input.credentials.secretAccessKey}`);
|
|
260
|
-
for (const part of [day, input.region, service, 'aws4_request']) key = await hmac(key, part);
|
|
261
|
-
const signature = toHex(await hmac(key, stringToSign));
|
|
262
|
-
|
|
303
|
+
const { stringToSign, signature } = await signatureOver(input, date, canonicalRequest);
|
|
304
|
+
const scope = scopeOf(input, date.slice(0, 8));
|
|
263
305
|
const sent = Object.fromEntries(headers);
|
|
264
306
|
sent['authorization'] =
|
|
265
307
|
`${ALGORITHM} Credential=${input.credentials.accessKeyId}/${scope}, SignedHeaders=${signedHeaders}, Signature=${signature}`;
|
|
@@ -273,3 +315,57 @@ export async function signAwsRequest(input: SignAwsRequestInput): Promise<Signed
|
|
|
273
315
|
signature,
|
|
274
316
|
};
|
|
275
317
|
}
|
|
318
|
+
|
|
319
|
+
/**
|
|
320
|
+
* Sign one request with SigV4 in its query, for a holder with no credentials. The payload is
|
|
321
|
+
* `UNSIGNED-PAYLOAD`, because the holder's body is not this process's to hash; a signed
|
|
322
|
+
* `content-length` is what bounds it instead, since the provider recomputes the signature from the
|
|
323
|
+
* header the request really carries.
|
|
324
|
+
*/
|
|
325
|
+
export async function presignAwsUrl(input: PresignAwsUrlInput): Promise<PresignedAwsUrl> {
|
|
326
|
+
assertInput(input);
|
|
327
|
+
const expires = finiteCount('presignAwsUrl', 'expiresInSeconds', input.expiresInSeconds, 1);
|
|
328
|
+
assert(
|
|
329
|
+
expires <= MAX_AWS_PRESIGN_SECONDS,
|
|
330
|
+
`presignAwsUrl was handed expiresInSeconds ${expires}, past SigV4's limit of ${MAX_AWS_PRESIGN_SECONDS}`,
|
|
331
|
+
`expiresInSeconds: ${MAX_AWS_PRESIGN_SECONDS} # seven days, the longest a presigned URL can live`,
|
|
332
|
+
);
|
|
333
|
+
const url = parseUrl(input.url);
|
|
334
|
+
const date = amzDate(input.clock ?? systemClock);
|
|
335
|
+
const caller: Record<string, string> = {};
|
|
336
|
+
for (const [name, value] of Object.entries(input.headers ?? {})) {
|
|
337
|
+
if (!SIGNER_OWNED.has(name.trim().toLowerCase())) caller[name] = value;
|
|
338
|
+
}
|
|
339
|
+
const hasHost = Object.keys(caller).some((name) => name.trim().toLowerCase() === 'host');
|
|
340
|
+
const headers = canonicalHeaders({ ...caller, ...(hasHost ? {} : { host: url.host }) });
|
|
341
|
+
const signedHeaders = [...headers.keys()].join(';');
|
|
342
|
+
const token = input.credentials.sessionToken;
|
|
343
|
+
const params = new URLSearchParams(url.search);
|
|
344
|
+
params.set('X-Amz-Algorithm', ALGORITHM);
|
|
345
|
+
params.set(
|
|
346
|
+
'X-Amz-Credential',
|
|
347
|
+
`${input.credentials.accessKeyId}/${scopeOf(input, date.slice(0, 8))}`,
|
|
348
|
+
);
|
|
349
|
+
params.set('X-Amz-Date', date);
|
|
350
|
+
params.set('X-Amz-Expires', String(expires));
|
|
351
|
+
params.set('X-Amz-SignedHeaders', signedHeaders);
|
|
352
|
+
if (token !== undefined && token !== '') params.set('X-Amz-Security-Token', token);
|
|
353
|
+
const query = canonicalQuery(params.toString().replaceAll('+', '%20'));
|
|
354
|
+
const doubleEncode = input.doubleEncodePath ?? input.service !== 's3';
|
|
355
|
+
const canonicalRequest = [
|
|
356
|
+
input.method,
|
|
357
|
+
canonicalPath(url.pathname, doubleEncode),
|
|
358
|
+
query,
|
|
359
|
+
...[...headers].map(([name, value]) => `${name}:${value}`),
|
|
360
|
+
'',
|
|
361
|
+
signedHeaders,
|
|
362
|
+
UNSIGNED_PAYLOAD,
|
|
363
|
+
].join('\n');
|
|
364
|
+
const { stringToSign, signature } = await signatureOver(input, date, canonicalRequest);
|
|
365
|
+
return {
|
|
366
|
+
url: `${url.origin}${canonicalPath(url.pathname, false)}?${query}&X-Amz-Signature=${signature}`,
|
|
367
|
+
canonicalRequest,
|
|
368
|
+
stringToSign,
|
|
369
|
+
signature,
|
|
370
|
+
};
|
|
371
|
+
}
|
package/src/breaker.ts
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
// Single responsibility: the circuit breaker's state machine, and nothing about what counts as a
|
|
2
|
+
// failure — the TRIGGER is the caller's (`classify`, or `failed()` called by hand). Four apps wrote
|
|
3
|
+
// this machine and disagreed only on the trigger; `@ultimat3/db`'s replica router held a private
|
|
4
|
+
// copy. Per process, by design: a shared row would make the breaker depend on the slow thing.
|
|
5
|
+
|
|
6
|
+
import type { Clock } from './clock';
|
|
7
|
+
import { systemClock } from './clock';
|
|
8
|
+
import { declaredErrorRetry, registerErrorRetry } from './error-retry';
|
|
9
|
+
import { isUltimateError, UltimateError } from './errors';
|
|
10
|
+
import { finiteCount } from './finite-option';
|
|
11
|
+
import { recordBreakerTransition } from './runtime-metrics';
|
|
12
|
+
|
|
13
|
+
// Here, not in `error-retry.ts`'s core table, which rides every browser page: a breaker is server
|
|
14
|
+
// work. Open names the time left in `retryAfterSeconds`, the flight gate's spelling.
|
|
15
|
+
registerErrorRetry({ X_BREAKER_OPEN: 'retry-after' });
|
|
16
|
+
|
|
17
|
+
export type BreakerState = 'closed' | 'open' | 'half-open';
|
|
18
|
+
|
|
19
|
+
/** What one settled call says about the dependency. `ignore` says nothing: a caller's abort. */
|
|
20
|
+
export type BreakerVerdict = 'failure' | 'success' | 'ignore';
|
|
21
|
+
|
|
22
|
+
export interface BreakerOptions {
|
|
23
|
+
/** A bounded label chosen in code (`'resend'`, `'replica'`) — never a tenant or a URL. */
|
|
24
|
+
readonly name: string;
|
|
25
|
+
/** Consecutive failures that open it. */
|
|
26
|
+
readonly failures: number;
|
|
27
|
+
/** How long it stays open before one probe is let through, in ms. */
|
|
28
|
+
readonly cooldownMs: number;
|
|
29
|
+
/** The trigger. Default: `defaultBreakerClassify`. */
|
|
30
|
+
readonly classify?: ((error: unknown) => BreakerVerdict) | undefined;
|
|
31
|
+
/** Monotonic time only: a wall-clock step must not close a breaker. */
|
|
32
|
+
readonly clock?: Pick<Clock, 'monotonic'> | undefined;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* One admission. The breaker settles the half-open probe only through ITS ticket, so a call
|
|
37
|
+
* admitted before the trip — aborting, failing or answering late — never frees or closes it.
|
|
38
|
+
*/
|
|
39
|
+
export interface BreakerTicket {
|
|
40
|
+
/** Whether this admission is the half-open probe. */
|
|
41
|
+
readonly probe: boolean;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export interface Breaker {
|
|
45
|
+
readonly name: string;
|
|
46
|
+
/** May a call go out now: its ticket, or `undefined`. In `half-open` the one ticket IS the probe. */
|
|
47
|
+
admit(): BreakerTicket | undefined;
|
|
48
|
+
/** The dependency answered. In `half-open` only the probe's ticket closes it; `open` ignores it. */
|
|
49
|
+
succeeded(ticket?: BreakerTicket): void;
|
|
50
|
+
failed(ticket?: BreakerTicket): void;
|
|
51
|
+
/** The call ended for a reason that says nothing: the probe's ticket gives the probe back. */
|
|
52
|
+
released(ticket?: BreakerTicket): void;
|
|
53
|
+
/** Read-only: never consumes a probe. `half-open` once the cooldown has passed. */
|
|
54
|
+
state(): BreakerState;
|
|
55
|
+
/** The current run of consecutive failures. */
|
|
56
|
+
failures(): number;
|
|
57
|
+
/** This breaker's `classify`, the default applied when none was given. */
|
|
58
|
+
judge(error: unknown): BreakerVerdict;
|
|
59
|
+
/** `X_BREAKER_OPEN` with the time left, for a caller that refuses without `run`. */
|
|
60
|
+
refusal(): UltimateError;
|
|
61
|
+
run<T>(fn: () => Promise<T>): Promise<T>;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Retryable means the dependency did not answer: a failure. A caller going away is no statement
|
|
66
|
+
* about the dependency (`X_ABORTED`, `X_DRAINING`, checked first: the second is retryable). Any
|
|
67
|
+
* other outcome is an answer, refusals included — a 400 is a vendor that is up.
|
|
68
|
+
*/
|
|
69
|
+
export function defaultBreakerClassify(error: unknown): BreakerVerdict {
|
|
70
|
+
if (!isUltimateError(error)) return 'success';
|
|
71
|
+
if (error.code === 'X_ABORTED' || error.code === 'X_DRAINING') return 'ignore';
|
|
72
|
+
const retry = declaredErrorRetry(error.code);
|
|
73
|
+
return retry === 'retryable' || retry === 'retry-after' ? 'failure' : 'success';
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export function breakerOpen(name: string, retryAfterSeconds: number): UltimateError {
|
|
77
|
+
return new UltimateError({
|
|
78
|
+
code: 'X_BREAKER_OPEN',
|
|
79
|
+
cause: `the breaker "${name}" is open after consecutive failures, so this call was not sent`,
|
|
80
|
+
fix: `breaker({ name: '${name}', failures, cooldownMs }) # raise failures if this outage is not one; else wait ${String(retryAfterSeconds)} s and check the dependency "${name}" names`,
|
|
81
|
+
meta: { name, retryAfterSeconds },
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export function breaker(options: BreakerOptions): Breaker {
|
|
86
|
+
const limit = finiteCount('breaker()', 'failures', options.failures, 1);
|
|
87
|
+
const cooldownMs = finiteCount('breaker()', 'cooldownMs', options.cooldownMs, 1);
|
|
88
|
+
const clock = options.clock ?? systemClock;
|
|
89
|
+
const classify = options.classify ?? defaultBreakerClassify;
|
|
90
|
+
let current: BreakerState = 'closed';
|
|
91
|
+
let count = 0;
|
|
92
|
+
/** When it opened (`open`) or when the probe went out (`half-open`). */
|
|
93
|
+
let since = 0;
|
|
94
|
+
|
|
95
|
+
const move = (to: BreakerState): void => {
|
|
96
|
+
if (current === to) return;
|
|
97
|
+
current = to;
|
|
98
|
+
recordBreakerTransition(options.name, to);
|
|
99
|
+
};
|
|
100
|
+
const lapsed = (): boolean => clock.monotonic() - since >= cooldownMs;
|
|
101
|
+
|
|
102
|
+
const CLOSED_TICKET: BreakerTicket = Object.freeze({ probe: false });
|
|
103
|
+
/** The outstanding probe's ticket, compared by identity. */
|
|
104
|
+
let probe: BreakerTicket | undefined;
|
|
105
|
+
const isProbe = (ticket: BreakerTicket | undefined): boolean =>
|
|
106
|
+
ticket !== undefined && ticket === probe;
|
|
107
|
+
|
|
108
|
+
const self: Breaker = {
|
|
109
|
+
name: options.name,
|
|
110
|
+
admit() {
|
|
111
|
+
if (current === 'closed') return CLOSED_TICKET;
|
|
112
|
+
if (current === 'open' && !lapsed()) return undefined;
|
|
113
|
+
// Past an open cooldown, or a probe that never settled: its holder may be dead.
|
|
114
|
+
if (current === 'half-open' && probe !== undefined && !lapsed()) return undefined;
|
|
115
|
+
move('half-open');
|
|
116
|
+
probe = { probe: true };
|
|
117
|
+
since = clock.monotonic();
|
|
118
|
+
return probe;
|
|
119
|
+
},
|
|
120
|
+
succeeded(ticket) {
|
|
121
|
+
// A late answer from a call admitted BEFORE the trip proves nothing about now: only a
|
|
122
|
+
// closed breaker or the half-open probe's own ticket may close it.
|
|
123
|
+
if (current === 'open' || (current === 'half-open' && !isProbe(ticket))) return;
|
|
124
|
+
count = 0;
|
|
125
|
+
probe = undefined;
|
|
126
|
+
move('closed');
|
|
127
|
+
},
|
|
128
|
+
failed(ticket) {
|
|
129
|
+
count += 1;
|
|
130
|
+
if (current === 'half-open' && isProbe(ticket)) {
|
|
131
|
+
probe = undefined;
|
|
132
|
+
since = clock.monotonic();
|
|
133
|
+
move('open');
|
|
134
|
+
} else if (current === 'closed' && count >= limit) {
|
|
135
|
+
since = clock.monotonic();
|
|
136
|
+
move('open');
|
|
137
|
+
}
|
|
138
|
+
},
|
|
139
|
+
released(ticket) {
|
|
140
|
+
if (isProbe(ticket)) probe = undefined;
|
|
141
|
+
},
|
|
142
|
+
state() {
|
|
143
|
+
return current === 'open' && lapsed() ? 'half-open' : current;
|
|
144
|
+
},
|
|
145
|
+
failures: () => count,
|
|
146
|
+
judge: (error) => classify(error),
|
|
147
|
+
refusal() {
|
|
148
|
+
const leftMs = Math.max(0, cooldownMs - (clock.monotonic() - since));
|
|
149
|
+
return breakerOpen(options.name, Math.max(1, Math.ceil(leftMs / 1_000)));
|
|
150
|
+
},
|
|
151
|
+
async run<T>(fn: () => Promise<T>): Promise<T> {
|
|
152
|
+
const ticket = self.admit();
|
|
153
|
+
if (ticket === undefined) throw self.refusal();
|
|
154
|
+
let value: T;
|
|
155
|
+
try {
|
|
156
|
+
value = await fn();
|
|
157
|
+
} catch (error) {
|
|
158
|
+
settleWith(self, self.judge(error), ticket);
|
|
159
|
+
throw error;
|
|
160
|
+
}
|
|
161
|
+
self.succeeded(ticket);
|
|
162
|
+
return value;
|
|
163
|
+
},
|
|
164
|
+
};
|
|
165
|
+
return self;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** Record one verdict. Exported for a caller that classifies itself (`outbound()`). */
|
|
169
|
+
export function settleWith(
|
|
170
|
+
gate: Breaker,
|
|
171
|
+
verdict: BreakerVerdict,
|
|
172
|
+
ticket: BreakerTicket | undefined,
|
|
173
|
+
): void {
|
|
174
|
+
if (verdict === 'failure') gate.failed(ticket);
|
|
175
|
+
else if (verdict === 'success') gate.succeeded(ticket);
|
|
176
|
+
else gate.released(ticket);
|
|
177
|
+
}
|
package/src/client-dispatch.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* One dispatch of a `clientTransport` request
|
|
3
|
-
* the
|
|
4
|
-
* (the flight's) or what
|
|
2
|
+
* One dispatch of a `clientTransport` request — or of `clientStream`'s, whose body is handed back
|
|
3
|
+
* unread: the `RequestInit`, the fence's abort for a read, the network fault's code, and the
|
|
4
|
+
* ok/error split — nothing about how many times it happens (the flight's) or what it means.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
import type { ClientFlight, ClientRetry } from './client-flight';
|
|
@@ -14,6 +14,7 @@ import { CLIENT_BUILD_META } from './page-meta';
|
|
|
14
14
|
import type { RecordEnvelope } from './record-envelope';
|
|
15
15
|
import { RECORDS_HEADER } from './record-envelope';
|
|
16
16
|
import { outboundSlot, pageClient } from './record-sink';
|
|
17
|
+
import { noteRefusal } from './session-lost';
|
|
17
18
|
import { DATES_HEADER } from './wire-dates';
|
|
18
19
|
|
|
19
20
|
export type FetchLike = (input: string, init: RequestInit) => Promise<Response>;
|
|
@@ -21,6 +22,9 @@ export type FetchLike = (input: string, init: RequestInit) => Promise<Response>;
|
|
|
21
22
|
/** The header `@ultimat3/action`'s server reads to replay rather than re-run a write. */
|
|
22
23
|
export const IDEMPOTENCY_HEADER = 'idempotency-key';
|
|
23
24
|
|
|
25
|
+
/** On a keyed answer: `1` when it was replayed from the first response, `0` when this call ran. */
|
|
26
|
+
export const REPLAYED_HEADER = 'x-ultimate-replayed';
|
|
27
|
+
|
|
24
28
|
/**
|
|
25
29
|
* The build a client was served, on the request, and the build that answered, on the response.
|
|
26
30
|
* `@ultimat3/http` reads it into `ctx.clientBuildId` and stamps it back; `@ultimat3/action`'s typed
|
|
@@ -105,22 +109,15 @@ export async function dispatch(
|
|
|
105
109
|
const fence = read ? abortable([req.signal, flightSignal]) : undefined;
|
|
106
110
|
const signal = fence === undefined ? req.signal : fence.signal;
|
|
107
111
|
try {
|
|
108
|
-
const
|
|
112
|
+
const retryable = read || req.idempotencyKey !== undefined;
|
|
113
|
+
const response = await onTheWire(req, retryable, () =>
|
|
109
114
|
(req.fetchImpl ?? browserFetch)(req.url, initOf(req, signal)),
|
|
110
115
|
);
|
|
111
116
|
await req.onResponse?.(response);
|
|
112
117
|
// Read as TEXT once: a body is a single-use stream, and the failure path wants it too.
|
|
113
|
-
const text =
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
response.headers.get('retry-after'),
|
|
117
|
-
response.headers.get('date'),
|
|
118
|
-
);
|
|
119
|
-
throw (
|
|
120
|
-
req.decodeError?.(response.status, text, stated) ??
|
|
121
|
-
problemError(response.status, text, req.url, stated)
|
|
122
|
-
);
|
|
123
|
-
}
|
|
118
|
+
const text =
|
|
119
|
+
response.status === 204 ? '' : await onTheWire(req, retryable, () => response.text());
|
|
120
|
+
if (!response.ok) throw refusalOf(req, response, text, issued);
|
|
124
121
|
return {
|
|
125
122
|
text,
|
|
126
123
|
enveloped: response.headers.get(RECORDS_HEADER) === '1',
|
|
@@ -135,6 +132,68 @@ export async function dispatch(
|
|
|
135
132
|
}
|
|
136
133
|
}
|
|
137
134
|
|
|
135
|
+
/** What `clientStream` sends: a write's request, never a raw body, a flight or a retry. */
|
|
136
|
+
export type StreamDispatch = Pick<
|
|
137
|
+
TransportRequest,
|
|
138
|
+
| 'url'
|
|
139
|
+
| 'body'
|
|
140
|
+
| 'headers'
|
|
141
|
+
| 'signal'
|
|
142
|
+
| 'idempotencyKey'
|
|
143
|
+
| 'decodeError'
|
|
144
|
+
| 'onResponse'
|
|
145
|
+
| 'fetchImpl'
|
|
146
|
+
>;
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* A streamed answer's dispatch: the same `POST` init, credentials, outbound headers and refusal
|
|
150
|
+
* fence as `dispatch`, bounded by the caller's signal, and the 2xx `Response` handed back with
|
|
151
|
+
* its body UNREAD for the caller to consume frame by frame. A write, so no read fence, and never
|
|
152
|
+
* retryable — even keyed: a stream is not an idempotent replay.
|
|
153
|
+
*/
|
|
154
|
+
export async function dispatchStream(req: StreamDispatch): Promise<Response> {
|
|
155
|
+
const post: TransportRequest = {
|
|
156
|
+
...req,
|
|
157
|
+
method: 'POST',
|
|
158
|
+
headers: { accept: 'text/event-stream', ...req.headers },
|
|
159
|
+
};
|
|
160
|
+
const issued = pageClient().scope.epoch;
|
|
161
|
+
const response = await onTheWire(post, false, () =>
|
|
162
|
+
(req.fetchImpl ?? browserFetch)(req.url, initOf(post, req.signal)),
|
|
163
|
+
);
|
|
164
|
+
await req.onResponse?.(response);
|
|
165
|
+
if (!response.ok) {
|
|
166
|
+
throw refusalOf(post, response, await onTheWire(post, false, () => response.text()), issued);
|
|
167
|
+
}
|
|
168
|
+
return response;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* A non-2xx answer as the caller's refusal, noted against the session it was issued for: only
|
|
173
|
+
* that principal can have lost it. The answer's own URL, so a same-origin call redirected
|
|
174
|
+
* elsewhere is judged by where it was refused.
|
|
175
|
+
*/
|
|
176
|
+
function refusalOf(
|
|
177
|
+
req: TransportRequest,
|
|
178
|
+
response: Response,
|
|
179
|
+
text: string,
|
|
180
|
+
issued: number,
|
|
181
|
+
): UltimateError {
|
|
182
|
+
const stated = retryAfterSecondsOf(
|
|
183
|
+
response.headers.get('retry-after'),
|
|
184
|
+
response.headers.get('date'),
|
|
185
|
+
);
|
|
186
|
+
const refusal =
|
|
187
|
+
req.decodeError?.(response.status, text, stated) ??
|
|
188
|
+
problemError(response.status, text, req.url, stated);
|
|
189
|
+
const scope = pageClient().scope;
|
|
190
|
+
if (scope.epoch === issued) {
|
|
191
|
+
const url = response.url === '' ? req.url : response.url;
|
|
192
|
+
noteRefusal({ status: response.status, url, code: refusal.code }, scope.principal);
|
|
193
|
+
}
|
|
194
|
+
return refusal;
|
|
195
|
+
}
|
|
196
|
+
|
|
138
197
|
/**
|
|
139
198
|
* The network's half of a dispatch, and only that half. `fetch` — and a body read cut mid-stream —
|
|
140
199
|
* rejects with a bare `TypeError` when no response arrived; that is `failure: 'network'`. A caller's
|
|
@@ -144,7 +203,7 @@ export async function dispatch(
|
|
|
144
203
|
*/
|
|
145
204
|
async function onTheWire<T>(
|
|
146
205
|
req: TransportRequest,
|
|
147
|
-
|
|
206
|
+
retryable: boolean,
|
|
148
207
|
work: () => Promise<T>,
|
|
149
208
|
): Promise<T> {
|
|
150
209
|
try {
|
|
@@ -154,7 +213,7 @@ async function onTheWire<T>(
|
|
|
154
213
|
throw transportFailed(
|
|
155
214
|
'network',
|
|
156
215
|
`${req.method} ${req.url} produced no response — the network refused or dropped it`,
|
|
157
|
-
|
|
216
|
+
retryable ? 'retryable' : undefined,
|
|
158
217
|
{ url: req.url, method: req.method },
|
|
159
218
|
error,
|
|
160
219
|
);
|
package/src/client-flight.ts
CHANGED
|
@@ -21,8 +21,8 @@ import { UltimateError } from './errors';
|
|
|
21
21
|
import type { FlightGate, FlightGateLimits } from './flight-gate';
|
|
22
22
|
import { flightGate } from './flight-gate';
|
|
23
23
|
import { generationFence } from './generation-fence';
|
|
24
|
-
import type { RetryPolicy } from './retry';
|
|
25
|
-
import {
|
|
24
|
+
import type { RetryPolicy } from './retry-loop';
|
|
25
|
+
import { retryLoop } from './retry-loop';
|
|
26
26
|
import type { Scheduler } from './single-flight';
|
|
27
27
|
import { singleFlight } from './single-flight';
|
|
28
28
|
|
|
@@ -49,7 +49,7 @@ export const DEFAULT_CLIENT_RETRY: RetryPolicy = {
|
|
|
49
49
|
* no response at all.
|
|
50
50
|
*
|
|
51
51
|
* The default is the OPPOSITE of `retryDecision`'s, one file over, and that is the point.
|
|
52
|
-
* `retry.ts` sends a throw nobody classified again until the attempts run out — which for a
|
|
52
|
+
* `retry-loop.ts` sends a throw nobody classified again until the attempts run out — which for a
|
|
53
53
|
* client means a caller's own `AbortError` and a `TypeError` out of a mapper both get retried.
|
|
54
54
|
* `@ultimat3/ai` and `@ultimat3/db` each refused that executor outright over it; this predicate
|
|
55
55
|
* keeps the executor and inverts the default instead, and is a parameter (`transient`) so an app
|
|
@@ -239,7 +239,7 @@ export function clientFlight(options: ClientFlightOptions = {}): ClientFlight {
|
|
|
239
239
|
// it would replace a code, a cause and a runnable `fix:` with the fact that something retried.
|
|
240
240
|
let stopped: { readonly error: unknown } | undefined;
|
|
241
241
|
const sendAgain = transientFor(plan);
|
|
242
|
-
const answer = await
|
|
242
|
+
const answer = await retryLoop<T | typeof STOPPED>(
|
|
243
243
|
async (count) => {
|
|
244
244
|
try {
|
|
245
245
|
return await once(plan, signal, count);
|