@juno-ai/bind 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +102 -0
- package/contracts/index.d.ts +1 -0
- package/contracts/index.js +1 -0
- package/contracts/turn.d.ts +79 -0
- package/contracts/turn.js +39 -0
- package/index.d.ts +13 -0
- package/index.js +13 -0
- package/package.json +43 -0
- package/routing/canonical-model.d.ts +17 -0
- package/routing/canonical-model.js +22 -0
- package/routing/circuit-breaker.d.ts +67 -0
- package/routing/circuit-breaker.js +75 -0
- package/routing/errors.d.ts +95 -0
- package/routing/errors.js +141 -0
- package/routing/executor.d.ts +64 -0
- package/routing/executor.js +173 -0
- package/routing/index.d.ts +9 -0
- package/routing/index.js +9 -0
- package/routing/plan.d.ts +98 -0
- package/routing/plan.js +22 -0
- package/routing/planner.d.ts +36 -0
- package/routing/planner.js +102 -0
- package/routing/policy.d.ts +81 -0
- package/routing/policy.js +99 -0
- package/routing/transport.d.ts +40 -0
- package/routing/transport.js +1 -0
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
const PROPAGATE = Object.freeze({
|
|
2
|
+
sameEndpointRetry: false,
|
|
3
|
+
nextProvider: false,
|
|
4
|
+
fallbackModel: false,
|
|
5
|
+
breaker: "none",
|
|
6
|
+
propagate: true,
|
|
7
|
+
});
|
|
8
|
+
const DEFECT = Object.freeze({
|
|
9
|
+
sameEndpointRetry: true,
|
|
10
|
+
nextProvider: true,
|
|
11
|
+
fallbackModel: true,
|
|
12
|
+
breaker: "record_failure",
|
|
13
|
+
propagate: false,
|
|
14
|
+
});
|
|
15
|
+
const RETRIABLE_TRANSPORT = Object.freeze({
|
|
16
|
+
sameEndpointRetry: false,
|
|
17
|
+
nextProvider: true,
|
|
18
|
+
fallbackModel: true,
|
|
19
|
+
breaker: "record_failure",
|
|
20
|
+
propagate: false,
|
|
21
|
+
});
|
|
22
|
+
/**
|
|
23
|
+
* Request-shaped rejections (a 400, a moderation refusal): another provider
|
|
24
|
+
* may accept the request, so traverse — but the ENDPOINT is healthy, so the
|
|
25
|
+
* breaker must not count it. Otherwise one tenant's malformed/flagged prompt
|
|
26
|
+
* degrades the shared endpoint for everyone (see PRD §7.2).
|
|
27
|
+
*/
|
|
28
|
+
const REQUEST_SHAPED = Object.freeze({
|
|
29
|
+
sameEndpointRetry: false,
|
|
30
|
+
nextProvider: true,
|
|
31
|
+
fallbackModel: true,
|
|
32
|
+
breaker: "none",
|
|
33
|
+
propagate: false,
|
|
34
|
+
});
|
|
35
|
+
const CREDENTIAL_FAILURE = Object.freeze({
|
|
36
|
+
sameEndpointRetry: false,
|
|
37
|
+
nextProvider: true,
|
|
38
|
+
fallbackModel: true,
|
|
39
|
+
breaker: "open_immediately",
|
|
40
|
+
propagate: false,
|
|
41
|
+
});
|
|
42
|
+
/**
|
|
43
|
+
* The exhaustive failure → routing-behavior matrix (PRD §7.1). Pure; the
|
|
44
|
+
* executor applies it, the circuit breaker consumes its `breaker` effect.
|
|
45
|
+
*/
|
|
46
|
+
export function failureDisposition(error) {
|
|
47
|
+
switch (error.kind) {
|
|
48
|
+
case "aborted":
|
|
49
|
+
return PROPAGATE;
|
|
50
|
+
case "completion_defect":
|
|
51
|
+
return DEFECT;
|
|
52
|
+
case "network":
|
|
53
|
+
return RETRIABLE_TRANSPORT;
|
|
54
|
+
case "http":
|
|
55
|
+
switch (error.category) {
|
|
56
|
+
case "timeout":
|
|
57
|
+
case "rate_limit":
|
|
58
|
+
case "server_error":
|
|
59
|
+
return RETRIABLE_TRANSPORT;
|
|
60
|
+
case "provider_bad_request":
|
|
61
|
+
return REQUEST_SHAPED;
|
|
62
|
+
case "credential":
|
|
63
|
+
case "credits":
|
|
64
|
+
return CREDENTIAL_FAILURE;
|
|
65
|
+
case "client_error":
|
|
66
|
+
return PROPAGATE;
|
|
67
|
+
default: {
|
|
68
|
+
const _exhaustive = error.category;
|
|
69
|
+
throw new Error(`unknown http category: ${JSON.stringify(_exhaustive)}`);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
default: {
|
|
73
|
+
const _exhaustive = error;
|
|
74
|
+
throw new Error(`unknown attempt error: ${JSON.stringify(_exhaustive)}`);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Default HTTP status → category mapping. A transport may override per
|
|
80
|
+
* provider (e.g. OpenRouter reports credit exhaustion as 402; another
|
|
81
|
+
* provider may use 403 with a body marker) — this is only the neutral
|
|
82
|
+
* baseline.
|
|
83
|
+
*/
|
|
84
|
+
export function categorizeHttpStatus(statusCode) {
|
|
85
|
+
if (statusCode === 408)
|
|
86
|
+
return "timeout";
|
|
87
|
+
if (statusCode === 429)
|
|
88
|
+
return "rate_limit";
|
|
89
|
+
if (statusCode >= 500)
|
|
90
|
+
return "server_error";
|
|
91
|
+
if (statusCode === 401 || statusCode === 403)
|
|
92
|
+
return "credential";
|
|
93
|
+
if (statusCode === 402)
|
|
94
|
+
return "credits";
|
|
95
|
+
if (statusCode === 400)
|
|
96
|
+
return "provider_bad_request";
|
|
97
|
+
return "client_error";
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Whether a FRESH ATTEMPT LATER could plausibly succeed — the caller-facing
|
|
101
|
+
* "retriable" used to pick the most useful error when a whole plan fails
|
|
102
|
+
* (PRD §5.2 step 6: prefer a retriable primary-path error over a
|
|
103
|
+
* non-retriable stale fallback binding, so the host's queue-level retry
|
|
104
|
+
* still fires).
|
|
105
|
+
*
|
|
106
|
+
* Deliberately NOT the same notion as route-traversal eligibility: a
|
|
107
|
+
* provider-specific 400 traverses to the next provider (a different provider
|
|
108
|
+
* may accept the request shape — `failureDisposition`), but retrying the
|
|
109
|
+
* same exhausted plan later won't fix it, so it is not caller-retriable.
|
|
110
|
+
* Credential/credit failures likewise traverse but need operator action, not
|
|
111
|
+
* time.
|
|
112
|
+
*/
|
|
113
|
+
export function isRetriableAttemptError(error) {
|
|
114
|
+
switch (error.kind) {
|
|
115
|
+
case "aborted":
|
|
116
|
+
return false;
|
|
117
|
+
case "completion_defect":
|
|
118
|
+
case "network":
|
|
119
|
+
return true;
|
|
120
|
+
case "http":
|
|
121
|
+
switch (error.category) {
|
|
122
|
+
case "timeout":
|
|
123
|
+
case "rate_limit":
|
|
124
|
+
case "server_error":
|
|
125
|
+
return true;
|
|
126
|
+
case "provider_bad_request":
|
|
127
|
+
case "credential":
|
|
128
|
+
case "credits":
|
|
129
|
+
case "client_error":
|
|
130
|
+
return false;
|
|
131
|
+
default: {
|
|
132
|
+
const _exhaustive = error.category;
|
|
133
|
+
throw new Error(`unknown http category: ${JSON.stringify(_exhaustive)}`);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
default: {
|
|
137
|
+
const _exhaustive = error;
|
|
138
|
+
throw new Error(`unknown attempt error: ${JSON.stringify(_exhaustive)}`);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import type { RouteCandidate, RoutePlan } from "./plan";
|
|
2
|
+
import { type InferenceAttemptError, type RouteAttemptCursor } from "./errors";
|
|
3
|
+
import type { RouteCircuitBreaker } from "./circuit-breaker";
|
|
4
|
+
export type AttemptOutcome<T> = Readonly<{
|
|
5
|
+
kind: "success";
|
|
6
|
+
value: T;
|
|
7
|
+
}> | Readonly<{
|
|
8
|
+
kind: "failure";
|
|
9
|
+
error: InferenceAttemptError;
|
|
10
|
+
}>;
|
|
11
|
+
/**
|
|
12
|
+
* One provider request. The host's transport adapter performs the network
|
|
13
|
+
* call, measures duration, and classifies any failure into
|
|
14
|
+
* `InferenceAttemptError` (transports classify facts; the executor owns
|
|
15
|
+
* route order — PRD §7.1).
|
|
16
|
+
*/
|
|
17
|
+
export type AttemptFn<T> = (candidate: RouteCandidate, cursor: RouteAttemptCursor) => Promise<AttemptOutcome<T>>;
|
|
18
|
+
export type FallbackKind = "none" | "provider" | "model" | "provider_and_model";
|
|
19
|
+
export interface ExecutePlanOptions<T> {
|
|
20
|
+
readonly plan: RoutePlan;
|
|
21
|
+
readonly attempt: AttemptFn<T>;
|
|
22
|
+
/** Stamped into cursors; the caller owns structured-output retry loops. */
|
|
23
|
+
readonly structuredOutputAttempt?: number;
|
|
24
|
+
/** Same-endpoint retry budget for completion defects. Default 2 (PRD §5.2). */
|
|
25
|
+
readonly completionDefectRetries?: number;
|
|
26
|
+
/** Optional per-endpoint circuit breaker (PRD §7.2). */
|
|
27
|
+
readonly breaker?: RouteCircuitBreaker | null;
|
|
28
|
+
}
|
|
29
|
+
export type RouteExecutionResult<T> = Readonly<{
|
|
30
|
+
ok: true;
|
|
31
|
+
value: T;
|
|
32
|
+
served: RouteCandidate;
|
|
33
|
+
cursor: RouteAttemptCursor;
|
|
34
|
+
fallbackKind: FallbackKind;
|
|
35
|
+
attemptCount: number;
|
|
36
|
+
failures: readonly InferenceAttemptError[];
|
|
37
|
+
}> | Readonly<{
|
|
38
|
+
ok: false;
|
|
39
|
+
reason: "no_viable_endpoints";
|
|
40
|
+
attemptCount: 0;
|
|
41
|
+
failures: readonly [];
|
|
42
|
+
}> | Readonly<{
|
|
43
|
+
ok: false;
|
|
44
|
+
reason: "attempts_exhausted";
|
|
45
|
+
error: InferenceAttemptError;
|
|
46
|
+
attemptCount: number;
|
|
47
|
+
failures: readonly InferenceAttemptError[];
|
|
48
|
+
}>;
|
|
49
|
+
/**
|
|
50
|
+
* Drive one structured-output attempt over a frozen route plan with the
|
|
51
|
+
* normative loop nesting (PRD §5.2): model stage → provider candidate →
|
|
52
|
+
* same-endpoint retry. Pure with respect to I/O — every request goes through
|
|
53
|
+
* `attempt`, every clock read through the injected breaker.
|
|
54
|
+
*/
|
|
55
|
+
export declare function executeRoutePlan<T>(options: ExecutePlanOptions<T>): Promise<RouteExecutionResult<T>>;
|
|
56
|
+
/**
|
|
57
|
+
* Derive the fallback classification from an attempt cursor — the single
|
|
58
|
+
* shared derivation for hosts that attribute per-attempt (e.g. inference
|
|
59
|
+
* logging inside a transport adapter) before the executor returns.
|
|
60
|
+
*/
|
|
61
|
+
export declare function fallbackKindOfCursor(cursor: {
|
|
62
|
+
readonly stageIndex: number;
|
|
63
|
+
readonly candidateIndex: number;
|
|
64
|
+
}): FallbackKind;
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import { failureDisposition, isRetriableAttemptError, } from "./errors";
|
|
2
|
+
/**
|
|
3
|
+
* Drive one structured-output attempt over a frozen route plan with the
|
|
4
|
+
* normative loop nesting (PRD §5.2): model stage → provider candidate →
|
|
5
|
+
* same-endpoint retry. Pure with respect to I/O — every request goes through
|
|
6
|
+
* `attempt`, every clock read through the injected breaker.
|
|
7
|
+
*/
|
|
8
|
+
export async function executeRoutePlan(options) {
|
|
9
|
+
const structuredOutputAttempt = options.structuredOutputAttempt ?? 0;
|
|
10
|
+
const defectRetries = options.completionDefectRetries ?? 2;
|
|
11
|
+
const breaker = options.breaker ?? null;
|
|
12
|
+
const failures = [];
|
|
13
|
+
let attemptCount = 0;
|
|
14
|
+
let anyCandidateSeen = false;
|
|
15
|
+
for (let stageIndex = 0; stageIndex < options.plan.stages.length; stageIndex++) {
|
|
16
|
+
const stage = options.plan.stages[stageIndex];
|
|
17
|
+
if (stage === undefined)
|
|
18
|
+
continue;
|
|
19
|
+
for (let candidateIndex = 0; candidateIndex < stage.candidates.length; candidateIndex++) {
|
|
20
|
+
const candidate = stage.candidates[candidateIndex];
|
|
21
|
+
if (candidate === undefined)
|
|
22
|
+
continue;
|
|
23
|
+
anyCandidateSeen = true;
|
|
24
|
+
const breakerKey = {
|
|
25
|
+
providerId: candidate.providerId,
|
|
26
|
+
invocationModel: candidate.providerInvocationModel,
|
|
27
|
+
credentialSource: candidate.credentialSource,
|
|
28
|
+
credentialScope: candidate.breakerScope,
|
|
29
|
+
};
|
|
30
|
+
let halfOpenProbe = false;
|
|
31
|
+
if (breaker !== null) {
|
|
32
|
+
const admission = breaker.admit(breakerKey);
|
|
33
|
+
if (!admission.admitted)
|
|
34
|
+
continue;
|
|
35
|
+
halfOpenProbe = admission.halfOpenProbe;
|
|
36
|
+
}
|
|
37
|
+
for (let endpointAttempt = 0; endpointAttempt <= defectRetries; endpointAttempt++) {
|
|
38
|
+
const cursor = Object.freeze({
|
|
39
|
+
structuredOutputAttempt,
|
|
40
|
+
stageIndex,
|
|
41
|
+
candidateIndex,
|
|
42
|
+
endpointAttempt,
|
|
43
|
+
});
|
|
44
|
+
attemptCount += 1;
|
|
45
|
+
const outcome = await options.attempt(candidate, cursor);
|
|
46
|
+
if (outcome.kind === "success") {
|
|
47
|
+
breaker?.recordSuccess(breakerKey);
|
|
48
|
+
return {
|
|
49
|
+
ok: true,
|
|
50
|
+
value: outcome.value,
|
|
51
|
+
served: candidate,
|
|
52
|
+
cursor,
|
|
53
|
+
fallbackKind: fallbackKindFor(stage.kind, candidateIndex),
|
|
54
|
+
attemptCount,
|
|
55
|
+
failures,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
failures.push(outcome.error);
|
|
59
|
+
const disposition = failureDisposition(outcome.error);
|
|
60
|
+
switch (disposition.breaker) {
|
|
61
|
+
case "none":
|
|
62
|
+
// A breaker-invisible failure (abort, propagated client error)
|
|
63
|
+
// must still resolve a half-open probe, or the probe slot stays
|
|
64
|
+
// occupied forever and the endpoint is refused until restart.
|
|
65
|
+
if (halfOpenProbe)
|
|
66
|
+
breaker?.releaseProbe(breakerKey);
|
|
67
|
+
break;
|
|
68
|
+
case "record_failure":
|
|
69
|
+
breaker?.recordFailure(breakerKey, {
|
|
70
|
+
retryAfterMs: retryAfterOf(outcome.error),
|
|
71
|
+
});
|
|
72
|
+
break;
|
|
73
|
+
case "open_immediately":
|
|
74
|
+
breaker?.recordFailure(breakerKey, {
|
|
75
|
+
openImmediately: true,
|
|
76
|
+
retryAfterMs: retryAfterOf(outcome.error),
|
|
77
|
+
});
|
|
78
|
+
break;
|
|
79
|
+
default: {
|
|
80
|
+
const _exhaustive = disposition.breaker;
|
|
81
|
+
throw new Error(`unknown breaker effect: ${JSON.stringify(_exhaustive)}`);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
if (disposition.propagate) {
|
|
85
|
+
return {
|
|
86
|
+
ok: false,
|
|
87
|
+
reason: "attempts_exhausted",
|
|
88
|
+
error: outcome.error,
|
|
89
|
+
attemptCount,
|
|
90
|
+
failures,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
if (disposition.sameEndpointRetry && endpointAttempt < defectRetries) {
|
|
94
|
+
continue; // consume the same-endpoint defect budget
|
|
95
|
+
}
|
|
96
|
+
// ANTICIPATORY: unreachable under the current disposition matrix
|
|
97
|
+
// (every non-propagating disposition traverses fully — pinned by the
|
|
98
|
+
// matrix-invariant test in errors.test.ts). Kept so a future
|
|
99
|
+
// asymmetric disposition is honored without an executor change.
|
|
100
|
+
if (!disposition.nextProvider) {
|
|
101
|
+
// The disposition matrix forbids trying another provider for this
|
|
102
|
+
// model; move (at most) into the fallback-model stage.
|
|
103
|
+
if (!disposition.fallbackModel) {
|
|
104
|
+
const error = preferredFinalError(failures, options.plan.stages.length);
|
|
105
|
+
if (error !== null) {
|
|
106
|
+
return { ok: false, reason: "attempts_exhausted", error, attemptCount, failures };
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
candidateIndex = stage.candidates.length; // exit the candidate loop
|
|
110
|
+
}
|
|
111
|
+
if (!disposition.fallbackModel) {
|
|
112
|
+
stageIndex = options.plan.stages.length; // no fallback-model traversal
|
|
113
|
+
}
|
|
114
|
+
break; // leave the same-endpoint retry loop
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
if (!anyCandidateSeen && failures.length === 0) {
|
|
119
|
+
return { ok: false, reason: "no_viable_endpoints", attemptCount: 0, failures: [] };
|
|
120
|
+
}
|
|
121
|
+
const error = preferredFinalError(failures, options.plan.stages.length);
|
|
122
|
+
if (error === null) {
|
|
123
|
+
// Candidates existed but every one was skipped by the breaker.
|
|
124
|
+
return { ok: false, reason: "no_viable_endpoints", attemptCount: 0, failures: [] };
|
|
125
|
+
}
|
|
126
|
+
return { ok: false, reason: "attempts_exhausted", error, attemptCount, failures };
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Derive the fallback classification from an attempt cursor — the single
|
|
130
|
+
* shared derivation for hosts that attribute per-attempt (e.g. inference
|
|
131
|
+
* logging inside a transport adapter) before the executor returns.
|
|
132
|
+
*/
|
|
133
|
+
export function fallbackKindOfCursor(cursor) {
|
|
134
|
+
if (cursor.stageIndex > 0) {
|
|
135
|
+
return cursor.candidateIndex > 0 ? "provider_and_model" : "model";
|
|
136
|
+
}
|
|
137
|
+
return cursor.candidateIndex > 0 ? "provider" : "none";
|
|
138
|
+
}
|
|
139
|
+
function fallbackKindFor(stageKind, candidateIndex) {
|
|
140
|
+
const providerFallback = candidateIndex > 0;
|
|
141
|
+
switch (stageKind) {
|
|
142
|
+
case "primary":
|
|
143
|
+
return providerFallback ? "provider" : "none";
|
|
144
|
+
case "fallback_model":
|
|
145
|
+
return providerFallback ? "provider_and_model" : "model";
|
|
146
|
+
default: {
|
|
147
|
+
const _exhaustive = stageKind;
|
|
148
|
+
throw new Error(`unknown stage kind: ${JSON.stringify(_exhaustive)}`);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
function retryAfterOf(error) {
|
|
153
|
+
return error.kind === "http" ? error.retryAfterMs : null;
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* PRD §5.2 step 6: when everything failed, prefer a retriable primary-stage
|
|
157
|
+
* error over a non-retriable error from the fallback stage (a stale fallback
|
|
158
|
+
* binding must not mask the actionable primary failure); otherwise report the
|
|
159
|
+
* final classified failure.
|
|
160
|
+
*/
|
|
161
|
+
function preferredFinalError(failures, stageCount) {
|
|
162
|
+
const last = failures.at(-1);
|
|
163
|
+
if (last === undefined)
|
|
164
|
+
return null;
|
|
165
|
+
if (stageCount < 2)
|
|
166
|
+
return last;
|
|
167
|
+
if (isRetriableAttemptError(last))
|
|
168
|
+
return last;
|
|
169
|
+
if (last.target.cursor.stageIndex === 0)
|
|
170
|
+
return last;
|
|
171
|
+
const primaryRetriable = failures.find((failure) => failure.target.cursor.stageIndex === 0 && isRetriableAttemptError(failure));
|
|
172
|
+
return primaryRetriable ?? last;
|
|
173
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export { canonicalModelIdSchema, providerIdSchema, type CanonicalModelId, type ProviderId, } from "./canonical-model";
|
|
2
|
+
export { ROUTE_POLICY_VERSION, providerPolicySchema, modelRouteRegistrySchema, parseModelRouteRegistry, resolveProviderPolicy, policyProviderOrder, type RoutePolicyVersion, type ProviderPolicy, type ModelRouteRegistry, } from "./policy";
|
|
3
|
+
export { INFERENCE_CAPABILITIES, planIsEmpty, type InferenceCapability, type CredentialSource, type InferenceRequirements, type CachedTokenSemantics, type ProviderPricingBasis, type RouteCandidate, type RouteStage, type RoutePlan, type RouteSkip, } from "./plan";
|
|
4
|
+
export { type TransportAvailability, type CandidateResolution, type PlannerTransport, } from "./transport";
|
|
5
|
+
export { buildRoutePlan, type RoutePlanRequest, type RoutePlanResult } from "./planner";
|
|
6
|
+
export { fallbackKindOfCursor } from "./executor";
|
|
7
|
+
export { failureDisposition, categorizeHttpStatus, isRetriableAttemptError, type RouteAttemptCursor, type AttemptTarget, type HttpFailureCategory, type InferenceAttemptError, type FailureDisposition, type BreakerEffect, } from "./errors";
|
|
8
|
+
export { createCircuitBreaker, type BreakerKey, type CircuitBreakerOptions, type EndpointAdmission, type RecordFailureOptions, type RouteCircuitBreaker, } from "./circuit-breaker";
|
|
9
|
+
export { executeRoutePlan, type AttemptOutcome, type AttemptFn, type FallbackKind, type ExecutePlanOptions, type RouteExecutionResult, } from "./executor";
|
package/routing/index.js
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export { canonicalModelIdSchema, providerIdSchema, } from "./canonical-model";
|
|
2
|
+
export { ROUTE_POLICY_VERSION, providerPolicySchema, modelRouteRegistrySchema, parseModelRouteRegistry, resolveProviderPolicy, policyProviderOrder, } from "./policy";
|
|
3
|
+
export { INFERENCE_CAPABILITIES, planIsEmpty, } from "./plan";
|
|
4
|
+
export {} from "./transport";
|
|
5
|
+
export { buildRoutePlan } from "./planner";
|
|
6
|
+
export { fallbackKindOfCursor } from "./executor";
|
|
7
|
+
export { failureDisposition, categorizeHttpStatus, isRetriableAttemptError, } from "./errors";
|
|
8
|
+
export { createCircuitBreaker, } from "./circuit-breaker";
|
|
9
|
+
export { executeRoutePlan, } from "./executor";
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import type { CanonicalModelId, ProviderId } from "./canonical-model";
|
|
2
|
+
import type { ProviderPolicy, RoutePolicyVersion } from "./policy";
|
|
3
|
+
/**
|
|
4
|
+
* Capabilities the planner checks before any network request (LLM Provider
|
|
5
|
+
* Routing PRD §5.1/§5.3). `max_completion_tokens` is modeled as a capability
|
|
6
|
+
* (a departure from the PRD draft, which listed the token-ceiling check in
|
|
7
|
+
* §5.3 prose but not in the union) so a ceiling violation produces the same
|
|
8
|
+
* `unsupported_capability` skip shape as any other mismatch.
|
|
9
|
+
*/
|
|
10
|
+
export declare const INFERENCE_CAPABILITIES: readonly ["chat_completions", "streaming", "tools", "structured_outputs", "image_input", "reasoning_effort", "file_parser_plugin", "service_tier", "max_completion_tokens"];
|
|
11
|
+
export type InferenceCapability = (typeof INFERENCE_CAPABILITIES)[number];
|
|
12
|
+
export type CredentialSource = "platform" | "tenant";
|
|
13
|
+
export interface InferenceRequirements {
|
|
14
|
+
readonly capabilities: ReadonlySet<InferenceCapability>;
|
|
15
|
+
readonly requestedMaxCompletionTokens: number | null;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* How a provider reports cached input tokens relative to `prompt_tokens`:
|
|
19
|
+
* `"subset"` — cached tokens are included in the prompt count (OpenAI);
|
|
20
|
+
* `"disjoint"` — cached tokens are reported alongside it and may exceed it
|
|
21
|
+
* (observed live on Grok served through Azure AI Foundry). Declared per
|
|
22
|
+
* binding, never inferred from the relative counts — a disjoint provider can
|
|
23
|
+
* legitimately report cached < prompt, which inference would misprice.
|
|
24
|
+
*/
|
|
25
|
+
export type CachedTokenSemantics = "subset" | "disjoint";
|
|
26
|
+
export type ProviderPricingBasis = Readonly<{
|
|
27
|
+
kind: "provider_reported";
|
|
28
|
+
}> | Readonly<{
|
|
29
|
+
kind: "configured_token_rates";
|
|
30
|
+
inputUsdPerM: number;
|
|
31
|
+
cachedInputUsdPerM: number;
|
|
32
|
+
outputUsdPerM: number;
|
|
33
|
+
cachedTokenSemantics: CachedTokenSemantics;
|
|
34
|
+
}>;
|
|
35
|
+
/**
|
|
36
|
+
* One concrete endpoint the executor may attempt. Secret-free by contract:
|
|
37
|
+
* no client, API key, base URL, prompt, or tenant secret — safe to inspect
|
|
38
|
+
* in tests and structured diagnostics (PRD §5.1).
|
|
39
|
+
*/
|
|
40
|
+
export interface RouteCandidate {
|
|
41
|
+
readonly providerId: ProviderId;
|
|
42
|
+
readonly canonicalModelId: CanonicalModelId;
|
|
43
|
+
readonly providerInvocationModel: string;
|
|
44
|
+
readonly credentialSource: CredentialSource;
|
|
45
|
+
readonly creditEligible: boolean;
|
|
46
|
+
readonly capabilities: ReadonlySet<InferenceCapability>;
|
|
47
|
+
/** null means the binding publishes no provider-side completion limit. */
|
|
48
|
+
readonly maxCompletionTokens: number | null;
|
|
49
|
+
readonly pricingBasis: ProviderPricingBasis;
|
|
50
|
+
/** Stable hash/version of non-secret binding data for diagnostics. */
|
|
51
|
+
readonly bindingFingerprint: string;
|
|
52
|
+
/**
|
|
53
|
+
* Optional non-secret credential identity used to scope circuit-breaker
|
|
54
|
+
* state beyond `credentialSource` (e.g. an opaque per-tenant tag for BYOK
|
|
55
|
+
* keys, so one tenant's revoked key never opens the circuit for other
|
|
56
|
+
* tenants). Never key material.
|
|
57
|
+
*/
|
|
58
|
+
readonly breakerScope?: string;
|
|
59
|
+
}
|
|
60
|
+
export interface RouteStage {
|
|
61
|
+
readonly kind: "primary" | "fallback_model";
|
|
62
|
+
readonly canonicalModelId: CanonicalModelId;
|
|
63
|
+
readonly policy: ProviderPolicy;
|
|
64
|
+
readonly candidates: readonly RouteCandidate[];
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* The immutable, versioned route plan built once per logical call and reused
|
|
68
|
+
* across structured-output retries (PRD §5.1). Contains no secrets and no
|
|
69
|
+
* request content.
|
|
70
|
+
*/
|
|
71
|
+
export interface RoutePlan {
|
|
72
|
+
readonly policyVersion: RoutePolicyVersion;
|
|
73
|
+
readonly requirements: InferenceRequirements;
|
|
74
|
+
readonly stages: readonly RouteStage[];
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* A capability/configuration omission recorded at plan time. Skips are
|
|
78
|
+
* planner diagnostics, not failed network attempts (PRD §7.1); `reason` is a
|
|
79
|
+
* controlled internal string, never a raw provider/parser message.
|
|
80
|
+
*/
|
|
81
|
+
export type RouteSkip = Readonly<{
|
|
82
|
+
kind: "transport_unavailable";
|
|
83
|
+
providerId: ProviderId;
|
|
84
|
+
canonicalModelId: CanonicalModelId;
|
|
85
|
+
reason: string;
|
|
86
|
+
}> | Readonly<{
|
|
87
|
+
kind: "unsupported_capability";
|
|
88
|
+
providerId: ProviderId;
|
|
89
|
+
canonicalModelId: CanonicalModelId;
|
|
90
|
+
capability: InferenceCapability;
|
|
91
|
+
}> | Readonly<{
|
|
92
|
+
kind: "invalid_binding";
|
|
93
|
+
providerId: ProviderId;
|
|
94
|
+
canonicalModelId: CanonicalModelId;
|
|
95
|
+
reason: string;
|
|
96
|
+
}>;
|
|
97
|
+
/** True when no stage has an attemptable candidate. */
|
|
98
|
+
export declare function planIsEmpty(plan: RoutePlan): boolean;
|
package/routing/plan.js
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Capabilities the planner checks before any network request (LLM Provider
|
|
3
|
+
* Routing PRD §5.1/§5.3). `max_completion_tokens` is modeled as a capability
|
|
4
|
+
* (a departure from the PRD draft, which listed the token-ceiling check in
|
|
5
|
+
* §5.3 prose but not in the union) so a ceiling violation produces the same
|
|
6
|
+
* `unsupported_capability` skip shape as any other mismatch.
|
|
7
|
+
*/
|
|
8
|
+
export const INFERENCE_CAPABILITIES = [
|
|
9
|
+
"chat_completions",
|
|
10
|
+
"streaming",
|
|
11
|
+
"tools",
|
|
12
|
+
"structured_outputs",
|
|
13
|
+
"image_input",
|
|
14
|
+
"reasoning_effort",
|
|
15
|
+
"file_parser_plugin",
|
|
16
|
+
"service_tier",
|
|
17
|
+
"max_completion_tokens",
|
|
18
|
+
];
|
|
19
|
+
/** True when no stage has an attemptable candidate. */
|
|
20
|
+
export function planIsEmpty(plan) {
|
|
21
|
+
return plan.stages.every((stage) => stage.candidates.length === 0);
|
|
22
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { CanonicalModelId, ProviderId } from "./canonical-model";
|
|
2
|
+
import { type ProviderPolicy } from "./policy";
|
|
3
|
+
import type { InferenceRequirements, RoutePlan, RouteSkip } from "./plan";
|
|
4
|
+
import type { PlannerTransport } from "./transport";
|
|
5
|
+
export interface RoutePlanRequest {
|
|
6
|
+
readonly primaryModel: CanonicalModelId;
|
|
7
|
+
/** A distinct fallback canonical model, or null. Equal to primary is ignored. */
|
|
8
|
+
readonly fallbackModel: CanonicalModelId | null;
|
|
9
|
+
readonly requirements: InferenceRequirements;
|
|
10
|
+
/**
|
|
11
|
+
* Policy resolution per canonical model — typically
|
|
12
|
+
* `resolveProviderPolicy(registry, model, boundProviders)` partially
|
|
13
|
+
* applied by the host. Injected as a function so the planner never reads
|
|
14
|
+
* configuration itself.
|
|
15
|
+
*/
|
|
16
|
+
readonly policyFor: (model: CanonicalModelId) => ProviderPolicy;
|
|
17
|
+
/**
|
|
18
|
+
* Composed transport registry. Map lookup by the provider ids the policy
|
|
19
|
+
* names — registration/iteration order never determines priority
|
|
20
|
+
* (PRD §4.3).
|
|
21
|
+
*/
|
|
22
|
+
readonly transports: ReadonlyMap<ProviderId, PlannerTransport>;
|
|
23
|
+
}
|
|
24
|
+
export interface RoutePlanResult {
|
|
25
|
+
readonly plan: RoutePlan;
|
|
26
|
+
/** Configuration/capability omissions recorded while planning (PRD §7.1). */
|
|
27
|
+
readonly skips: readonly RouteSkip[];
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Deterministically expand a canonical primary/fallback selection into an
|
|
31
|
+
* immutable, ordered route plan (LLM Provider Routing PRD §5.1). The only
|
|
32
|
+
* ordering inputs are the two model stages and each stage's policy provider
|
|
33
|
+
* order. Identical `(provider, invocation model)` endpoints are de-duplicated
|
|
34
|
+
* across the whole plan, first occurrence wins.
|
|
35
|
+
*/
|
|
36
|
+
export declare function buildRoutePlan(request: RoutePlanRequest): RoutePlanResult;
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { ROUTE_POLICY_VERSION, policyProviderOrder, } from "./policy";
|
|
2
|
+
/**
|
|
3
|
+
* Deterministically expand a canonical primary/fallback selection into an
|
|
4
|
+
* immutable, ordered route plan (LLM Provider Routing PRD §5.1). The only
|
|
5
|
+
* ordering inputs are the two model stages and each stage's policy provider
|
|
6
|
+
* order. Identical `(provider, invocation model)` endpoints are de-duplicated
|
|
7
|
+
* across the whole plan, first occurrence wins.
|
|
8
|
+
*/
|
|
9
|
+
export function buildRoutePlan(request) {
|
|
10
|
+
const skips = [];
|
|
11
|
+
const seenEndpoints = new Set();
|
|
12
|
+
const stageModels = request.fallbackModel !== null && request.fallbackModel !== request.primaryModel
|
|
13
|
+
? [
|
|
14
|
+
{ kind: "primary", model: request.primaryModel },
|
|
15
|
+
{ kind: "fallback_model", model: request.fallbackModel },
|
|
16
|
+
]
|
|
17
|
+
: [{ kind: "primary", model: request.primaryModel }];
|
|
18
|
+
const stages = [];
|
|
19
|
+
for (const { kind, model } of stageModels) {
|
|
20
|
+
const policy = request.policyFor(model);
|
|
21
|
+
const candidates = [];
|
|
22
|
+
for (const providerId of policyProviderOrder(policy)) {
|
|
23
|
+
const transport = request.transports.get(providerId);
|
|
24
|
+
if (transport === undefined) {
|
|
25
|
+
skips.push({
|
|
26
|
+
kind: "transport_unavailable",
|
|
27
|
+
providerId,
|
|
28
|
+
canonicalModelId: model,
|
|
29
|
+
reason: "no transport registered for provider",
|
|
30
|
+
});
|
|
31
|
+
continue;
|
|
32
|
+
}
|
|
33
|
+
const availability = transport.getAvailability();
|
|
34
|
+
if (!availability.available) {
|
|
35
|
+
skips.push({
|
|
36
|
+
kind: "transport_unavailable",
|
|
37
|
+
providerId,
|
|
38
|
+
canonicalModelId: model,
|
|
39
|
+
reason: availability.reason,
|
|
40
|
+
});
|
|
41
|
+
continue;
|
|
42
|
+
}
|
|
43
|
+
const resolution = transport.resolveCandidate(model, request.requirements);
|
|
44
|
+
switch (resolution.kind) {
|
|
45
|
+
case "unserved":
|
|
46
|
+
continue;
|
|
47
|
+
case "skip":
|
|
48
|
+
skips.push(resolution.skip);
|
|
49
|
+
continue;
|
|
50
|
+
case "candidate":
|
|
51
|
+
break;
|
|
52
|
+
default: {
|
|
53
|
+
const _exhaustive = resolution;
|
|
54
|
+
throw new Error(`unknown resolution: ${JSON.stringify(_exhaustive)}`);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
const candidate = resolution.candidate;
|
|
58
|
+
const missing = capabilityGaps(candidate, request.requirements);
|
|
59
|
+
if (missing.length > 0) {
|
|
60
|
+
for (const capability of missing) {
|
|
61
|
+
skips.push({
|
|
62
|
+
kind: "unsupported_capability",
|
|
63
|
+
providerId,
|
|
64
|
+
canonicalModelId: model,
|
|
65
|
+
capability,
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
const endpointKey = `${candidate.providerId}\u0000${candidate.providerInvocationModel}`;
|
|
71
|
+
if (seenEndpoints.has(endpointKey))
|
|
72
|
+
continue;
|
|
73
|
+
seenEndpoints.add(endpointKey);
|
|
74
|
+
candidates.push(Object.freeze(candidate));
|
|
75
|
+
}
|
|
76
|
+
stages.push(Object.freeze({
|
|
77
|
+
kind,
|
|
78
|
+
canonicalModelId: model,
|
|
79
|
+
policy,
|
|
80
|
+
candidates: Object.freeze(candidates),
|
|
81
|
+
}));
|
|
82
|
+
}
|
|
83
|
+
const plan = Object.freeze({
|
|
84
|
+
policyVersion: ROUTE_POLICY_VERSION,
|
|
85
|
+
requirements: request.requirements,
|
|
86
|
+
stages: Object.freeze(stages),
|
|
87
|
+
});
|
|
88
|
+
return { plan, skips: Object.freeze(skips) };
|
|
89
|
+
}
|
|
90
|
+
function capabilityGaps(candidate, requirements) {
|
|
91
|
+
const missing = [];
|
|
92
|
+
for (const capability of requirements.capabilities) {
|
|
93
|
+
if (!candidate.capabilities.has(capability))
|
|
94
|
+
missing.push(capability);
|
|
95
|
+
}
|
|
96
|
+
if (requirements.requestedMaxCompletionTokens !== null &&
|
|
97
|
+
candidate.maxCompletionTokens !== null &&
|
|
98
|
+
requirements.requestedMaxCompletionTokens > candidate.maxCompletionTokens) {
|
|
99
|
+
missing.push("max_completion_tokens");
|
|
100
|
+
}
|
|
101
|
+
return missing;
|
|
102
|
+
}
|