@juspay/neurolink 11.5.0 → 11.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -6
- package/dist/browser/neurolink.min.js +371 -371
- package/dist/lib/providers/amazonBedrock/client.js +14 -5
- package/dist/lib/providers/amazonBedrock/inferenceProfile.d.ts +72 -0
- package/dist/lib/providers/amazonBedrock/inferenceProfile.js +186 -0
- package/dist/providers/amazonBedrock/client.js +14 -5
- package/dist/providers/amazonBedrock/inferenceProfile.d.ts +72 -0
- package/dist/providers/amazonBedrock/inferenceProfile.js +185 -0
- package/package.json +7 -6
|
@@ -3,6 +3,7 @@ import path from "path";
|
|
|
3
3
|
import { createAnalytics } from "../../core/analytics.js";
|
|
4
4
|
import { BaseProvider } from "../../core/baseProvider.js";
|
|
5
5
|
import { DEFAULT_MAX_STEPS } from "../../core/constants.js";
|
|
6
|
+
import { withInferenceProfileFallback } from "./inferenceProfile.js";
|
|
6
7
|
import { AuthenticationError, ProviderError, RateLimitError, } from "../../types/index.js";
|
|
7
8
|
import { classifyProviderError } from "../../utils/errorClassifier.js";
|
|
8
9
|
import { isAbortError, withTimeout } from "../../utils/errorHandling.js";
|
|
@@ -370,8 +371,6 @@ export class AmazonBedrockProvider extends BaseProvider {
|
|
|
370
371
|
logger.info(` - Max tokens: ${commandInput.inferenceConfig?.maxTokens}`);
|
|
371
372
|
logger.info(` - Temperature: ${commandInput.inferenceConfig?.temperature}`);
|
|
372
373
|
logger.debug(`[AmazonBedrockProvider] Calling Bedrock with ${this.conversationHistory.length} messages and ${toolConfig?.tools?.length || 0} tools`);
|
|
373
|
-
// Create command and attempt API call
|
|
374
|
-
const command = new ConverseCommand(commandInput);
|
|
375
374
|
logger.debug("[Observability] Bedrock API request", {
|
|
376
375
|
model: commandInput.modelId,
|
|
377
376
|
region: region,
|
|
@@ -380,7 +379,15 @@ export class AmazonBedrockProvider extends BaseProvider {
|
|
|
380
379
|
maxTokens: commandInput.inferenceConfig?.maxTokens,
|
|
381
380
|
});
|
|
382
381
|
const apiCallStartTime = Date.now();
|
|
383
|
-
const response = await
|
|
382
|
+
const response = await withInferenceProfileFallback(commandInput.modelId ?? "",
|
|
383
|
+
// `this.region`, not the local `region` above: that one is
|
|
384
|
+
// best-effort for logging and stays "unknown" if the lookup
|
|
385
|
+
// throws. It must also match the id the streaming path keys its
|
|
386
|
+
// cache by, or a resolution found here is never reused there.
|
|
387
|
+
this.region, (effectiveModelId) => withTimeout(this.bedrockClient.send(new ConverseCommand({
|
|
388
|
+
...commandInput,
|
|
389
|
+
modelId: effectiveModelId,
|
|
390
|
+
})), 120_000, new Error("Bedrock API call timed out")));
|
|
384
391
|
const apiCallDuration = Date.now() - apiCallStartTime;
|
|
385
392
|
logger.debug("[Observability] Bedrock API response", {
|
|
386
393
|
model: commandInput.modelId,
|
|
@@ -1062,7 +1069,6 @@ export class AmazonBedrockProvider extends BaseProvider {
|
|
|
1062
1069
|
try {
|
|
1063
1070
|
logger.debug("[TRACE] streamingConversationLoop - testing first streaming call");
|
|
1064
1071
|
const commandInput = await this.prepareStreamCommand(options);
|
|
1065
|
-
const command = new ConverseStreamCommand(commandInput);
|
|
1066
1072
|
logger.debug("[Observability] Bedrock streaming API request", {
|
|
1067
1073
|
model: commandInput.modelId,
|
|
1068
1074
|
messageCount: commandInput.messages?.length || 0,
|
|
@@ -1073,7 +1079,10 @@ export class AmazonBedrockProvider extends BaseProvider {
|
|
|
1073
1079
|
"bedrock.tool_count": commandInput.toolConfig?.tools?.length || 0,
|
|
1074
1080
|
});
|
|
1075
1081
|
const streamStartTime = Date.now();
|
|
1076
|
-
const response = await withTimeout(this.bedrockClient.send(
|
|
1082
|
+
const response = await withInferenceProfileFallback(commandInput.modelId ?? "", this.region, (effectiveModelId) => withTimeout(this.bedrockClient.send(new ConverseStreamCommand({
|
|
1083
|
+
...commandInput,
|
|
1084
|
+
modelId: effectiveModelId,
|
|
1085
|
+
})), 120_000, new Error("Bedrock streaming API call timed out")));
|
|
1077
1086
|
logger.debug("[Observability] Bedrock streaming API connection established", {
|
|
1078
1087
|
model: commandInput.modelId,
|
|
1079
1088
|
durationMs: Date.now() - streamStartTime,
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bedrock cross-region inference profile resolution.
|
|
3
|
+
*
|
|
4
|
+
* Most current Bedrock models cannot be invoked by their bare model id. Each
|
|
5
|
+
* model card carries a Regional Availability table with three columns, and for
|
|
6
|
+
* a great many model/region pairs the In-Region column is "No" — Claude Opus
|
|
7
|
+
* 4.5 is No in every region AWS lists, Sonnet 4.6 is Yes only in eu-west-2.
|
|
8
|
+
* Those models are reachable only through a cross-region inference profile: the
|
|
9
|
+
* same id with a geography prefix (`us.`, `eu.`, `au.`, `jp.`, `apac.`) or the
|
|
10
|
+
* worldwide `global.` prefix.
|
|
11
|
+
*
|
|
12
|
+
* Sending the bare id where In-Region is No fails with a ValidationException
|
|
13
|
+
* telling you to use an inference profile. This module detects that specific
|
|
14
|
+
* failure and works out which prefixed id to retry with.
|
|
15
|
+
*
|
|
16
|
+
* ## Why prefixes are derived rather than tabulated
|
|
17
|
+
*
|
|
18
|
+
* The obvious implementation — a table of model to supported prefixes — is the
|
|
19
|
+
* one to avoid. The prefix set is per-model, not per-region: from the same
|
|
20
|
+
* caller region `ap-northeast-1`, Claude Opus 4.5 accepts only `global.`,
|
|
21
|
+
* Sonnet 4.5 also accepts `jp.`, and Haiku 4.5 works bare. A static table
|
|
22
|
+
* would need a row per model per region and would be wrong the day a model
|
|
23
|
+
* ships. AWS itself moved this data out of a central page and onto each
|
|
24
|
+
* model's detail page for the same reason.
|
|
25
|
+
*
|
|
26
|
+
* So resolution is empirical: on the specific error, try the geography that
|
|
27
|
+
* covers the caller's region, then `global.`, and remember what worked.
|
|
28
|
+
*
|
|
29
|
+
* `bedrock:ListInferenceProfiles` would give an authoritative answer, but it
|
|
30
|
+
* is a control-plane call needing a separate SDK client and an IAM permission
|
|
31
|
+
* a caller with plain inference access will not necessarily hold. Deriving
|
|
32
|
+
* candidates needs neither, and costs at most two extra attempts once per
|
|
33
|
+
* model/region pair.
|
|
34
|
+
*/
|
|
35
|
+
/**
|
|
36
|
+
* Does this error mean "this model needs an inference profile in this region"?
|
|
37
|
+
*
|
|
38
|
+
* Deliberately narrow. A ValidationException covers many unrelated causes —
|
|
39
|
+
* a malformed body, an unknown parameter, a model the account has no access to
|
|
40
|
+
* — and retrying those with a different model id would turn one clear failure
|
|
41
|
+
* into two confusing ones. Both the exception name and the distinctive phrase
|
|
42
|
+
* must match.
|
|
43
|
+
*
|
|
44
|
+
* The phrase is matched loosely enough to survive AWS rewording the sentence
|
|
45
|
+
* around it, and the retry is a no-op when nothing resolves, so a miss here
|
|
46
|
+
* costs nothing beyond the original error surfacing unchanged — which is the
|
|
47
|
+
* behaviour without this module at all.
|
|
48
|
+
*/
|
|
49
|
+
export declare function isInferenceProfileRequiredError(error: unknown): boolean;
|
|
50
|
+
/**
|
|
51
|
+
* Candidate ids to retry, most specific first: the caller's own geography,
|
|
52
|
+
* then worldwide.
|
|
53
|
+
*
|
|
54
|
+
* Returns an empty array when the id already carries a prefix — a prefixed id
|
|
55
|
+
* that still fails is a real error, not something to re-prefix.
|
|
56
|
+
*/
|
|
57
|
+
export declare function inferenceProfileCandidates(modelId: string, region: string): string[];
|
|
58
|
+
/** The id that previously worked for this model in this region, if any. */
|
|
59
|
+
export declare function getResolvedModelId(modelId: string, region: string): string | undefined;
|
|
60
|
+
export declare function rememberResolvedModelId(modelId: string, region: string, resolvedId: string): void;
|
|
61
|
+
/** Exposed so tests can start from a known state. */
|
|
62
|
+
export declare function clearResolvedModelIds(): void;
|
|
63
|
+
/**
|
|
64
|
+
* Run `send` against the given model id, falling back to inference-profile
|
|
65
|
+
* ids if — and only if — Bedrock says the bare id needs one.
|
|
66
|
+
*
|
|
67
|
+
* A previously resolved id is used straight away. Any error other than the
|
|
68
|
+
* inference-profile one propagates untouched on the first attempt, so this
|
|
69
|
+
* cannot mask an unrelated failure or silently change behaviour for a caller
|
|
70
|
+
* whose bare ids already work.
|
|
71
|
+
*/
|
|
72
|
+
export declare function withInferenceProfileFallback<T>(modelId: string, region: string, send: (effectiveModelId: string) => Promise<T>): Promise<T>;
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bedrock cross-region inference profile resolution.
|
|
3
|
+
*
|
|
4
|
+
* Most current Bedrock models cannot be invoked by their bare model id. Each
|
|
5
|
+
* model card carries a Regional Availability table with three columns, and for
|
|
6
|
+
* a great many model/region pairs the In-Region column is "No" — Claude Opus
|
|
7
|
+
* 4.5 is No in every region AWS lists, Sonnet 4.6 is Yes only in eu-west-2.
|
|
8
|
+
* Those models are reachable only through a cross-region inference profile: the
|
|
9
|
+
* same id with a geography prefix (`us.`, `eu.`, `au.`, `jp.`, `apac.`) or the
|
|
10
|
+
* worldwide `global.` prefix.
|
|
11
|
+
*
|
|
12
|
+
* Sending the bare id where In-Region is No fails with a ValidationException
|
|
13
|
+
* telling you to use an inference profile. This module detects that specific
|
|
14
|
+
* failure and works out which prefixed id to retry with.
|
|
15
|
+
*
|
|
16
|
+
* ## Why prefixes are derived rather than tabulated
|
|
17
|
+
*
|
|
18
|
+
* The obvious implementation — a table of model to supported prefixes — is the
|
|
19
|
+
* one to avoid. The prefix set is per-model, not per-region: from the same
|
|
20
|
+
* caller region `ap-northeast-1`, Claude Opus 4.5 accepts only `global.`,
|
|
21
|
+
* Sonnet 4.5 also accepts `jp.`, and Haiku 4.5 works bare. A static table
|
|
22
|
+
* would need a row per model per region and would be wrong the day a model
|
|
23
|
+
* ships. AWS itself moved this data out of a central page and onto each
|
|
24
|
+
* model's detail page for the same reason.
|
|
25
|
+
*
|
|
26
|
+
* So resolution is empirical: on the specific error, try the geography that
|
|
27
|
+
* covers the caller's region, then `global.`, and remember what worked.
|
|
28
|
+
*
|
|
29
|
+
* `bedrock:ListInferenceProfiles` would give an authoritative answer, but it
|
|
30
|
+
* is a control-plane call needing a separate SDK client and an IAM permission
|
|
31
|
+
* a caller with plain inference access will not necessarily hold. Deriving
|
|
32
|
+
* candidates needs neither, and costs at most two extra attempts once per
|
|
33
|
+
* model/region pair.
|
|
34
|
+
*/
|
|
35
|
+
import { logger } from "../../utils/logger.js";
|
|
36
|
+
/**
|
|
37
|
+
* Geography prefixes, in the order AWS documents them. `global.` is not a
|
|
38
|
+
* geography — it routes anywhere and is tried last, as the broadest option.
|
|
39
|
+
*/
|
|
40
|
+
const GLOBAL_PREFIX = "global";
|
|
41
|
+
/**
|
|
42
|
+
* Maps an AWS region to the geography whose inference profile covers it.
|
|
43
|
+
*
|
|
44
|
+
* Derived from the Geo inference tables on the model cards: the US geography
|
|
45
|
+
* covers `us-*` plus the Canadian regions, EU covers `eu-*` plus Israel, the
|
|
46
|
+
* Middle East and Africa, Japan covers the two Japanese regions, and Australia
|
|
47
|
+
* covers Sydney, Melbourne and New Zealand. Everything else in Asia Pacific
|
|
48
|
+
* falls under `apac`.
|
|
49
|
+
*/
|
|
50
|
+
function geoPrefixForRegion(region) {
|
|
51
|
+
if (/^(us|ca)-/.test(region)) {
|
|
52
|
+
return "us";
|
|
53
|
+
}
|
|
54
|
+
if (/^(eu|il|me|af)-/.test(region)) {
|
|
55
|
+
return "eu";
|
|
56
|
+
}
|
|
57
|
+
if (/^ap-northeast-[13]$/.test(region)) {
|
|
58
|
+
return "jp";
|
|
59
|
+
}
|
|
60
|
+
if (/^ap-southeast-[246]$/.test(region)) {
|
|
61
|
+
return "au";
|
|
62
|
+
}
|
|
63
|
+
if (/^ap-/.test(region)) {
|
|
64
|
+
return "apac";
|
|
65
|
+
}
|
|
66
|
+
return undefined;
|
|
67
|
+
}
|
|
68
|
+
/** True when `modelId` already carries a geography or global prefix. */
|
|
69
|
+
function hasProfilePrefix(modelId) {
|
|
70
|
+
return /^(us|eu|au|jp|apac|global)\./.test(modelId);
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Does this error mean "this model needs an inference profile in this region"?
|
|
74
|
+
*
|
|
75
|
+
* Deliberately narrow. A ValidationException covers many unrelated causes —
|
|
76
|
+
* a malformed body, an unknown parameter, a model the account has no access to
|
|
77
|
+
* — and retrying those with a different model id would turn one clear failure
|
|
78
|
+
* into two confusing ones. Both the exception name and the distinctive phrase
|
|
79
|
+
* must match.
|
|
80
|
+
*
|
|
81
|
+
* The phrase is matched loosely enough to survive AWS rewording the sentence
|
|
82
|
+
* around it, and the retry is a no-op when nothing resolves, so a miss here
|
|
83
|
+
* costs nothing beyond the original error surfacing unchanged — which is the
|
|
84
|
+
* behaviour without this module at all.
|
|
85
|
+
*/
|
|
86
|
+
export function isInferenceProfileRequiredError(error) {
|
|
87
|
+
if (typeof error !== "object" || error === null) {
|
|
88
|
+
return false;
|
|
89
|
+
}
|
|
90
|
+
const candidate = error;
|
|
91
|
+
const name = typeof candidate.name === "string" ? candidate.name : "";
|
|
92
|
+
const message = typeof candidate.message === "string" ? candidate.message : "";
|
|
93
|
+
if (!/ValidationException/i.test(name) &&
|
|
94
|
+
!/ValidationException/i.test(message)) {
|
|
95
|
+
return false;
|
|
96
|
+
}
|
|
97
|
+
return (/inference profile/i.test(message) && /on-demand throughput/i.test(message));
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Candidate ids to retry, most specific first: the caller's own geography,
|
|
101
|
+
* then worldwide.
|
|
102
|
+
*
|
|
103
|
+
* Returns an empty array when the id already carries a prefix — a prefixed id
|
|
104
|
+
* that still fails is a real error, not something to re-prefix.
|
|
105
|
+
*/
|
|
106
|
+
export function inferenceProfileCandidates(modelId, region) {
|
|
107
|
+
// An ARN already names a concrete model or inference-profile resource, and
|
|
108
|
+
// this provider accepts one as a model id (see `extractRegionFromArn`).
|
|
109
|
+
// Prefixing it yields a malformed identifier whose ValidationException would
|
|
110
|
+
// then replace the original, accurate error.
|
|
111
|
+
if (modelId.startsWith("arn:") || hasProfilePrefix(modelId)) {
|
|
112
|
+
return [];
|
|
113
|
+
}
|
|
114
|
+
const candidates = [];
|
|
115
|
+
const geo = geoPrefixForRegion(region);
|
|
116
|
+
if (geo) {
|
|
117
|
+
candidates.push(`${geo}.${modelId}`);
|
|
118
|
+
}
|
|
119
|
+
candidates.push(`${GLOBAL_PREFIX}.${modelId}`);
|
|
120
|
+
return candidates;
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Remembers which id actually worked, keyed by region and bare model id, so
|
|
124
|
+
* the probing happens once rather than on every call.
|
|
125
|
+
*/
|
|
126
|
+
const resolved = new Map();
|
|
127
|
+
function cacheKey(region, modelId) {
|
|
128
|
+
return `${region}::${modelId}`;
|
|
129
|
+
}
|
|
130
|
+
/** The id that previously worked for this model in this region, if any. */
|
|
131
|
+
export function getResolvedModelId(modelId, region) {
|
|
132
|
+
return resolved.get(cacheKey(region, modelId));
|
|
133
|
+
}
|
|
134
|
+
export function rememberResolvedModelId(modelId, region, resolvedId) {
|
|
135
|
+
resolved.set(cacheKey(region, modelId), resolvedId);
|
|
136
|
+
logger.debug("[Bedrock] Cached inference-profile resolution for subsequent calls", { region });
|
|
137
|
+
}
|
|
138
|
+
/** Exposed so tests can start from a known state. */
|
|
139
|
+
export function clearResolvedModelIds() {
|
|
140
|
+
resolved.clear();
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Run `send` against the given model id, falling back to inference-profile
|
|
144
|
+
* ids if — and only if — Bedrock says the bare id needs one.
|
|
145
|
+
*
|
|
146
|
+
* A previously resolved id is used straight away. Any error other than the
|
|
147
|
+
* inference-profile one propagates untouched on the first attempt, so this
|
|
148
|
+
* cannot mask an unrelated failure or silently change behaviour for a caller
|
|
149
|
+
* whose bare ids already work.
|
|
150
|
+
*/
|
|
151
|
+
export async function withInferenceProfileFallback(modelId, region, send) {
|
|
152
|
+
const cached = getResolvedModelId(modelId, region);
|
|
153
|
+
if (cached && cached !== modelId) {
|
|
154
|
+
return send(cached);
|
|
155
|
+
}
|
|
156
|
+
try {
|
|
157
|
+
return await send(modelId);
|
|
158
|
+
}
|
|
159
|
+
catch (error) {
|
|
160
|
+
if (!isInferenceProfileRequiredError(error)) {
|
|
161
|
+
throw error;
|
|
162
|
+
}
|
|
163
|
+
const candidates = inferenceProfileCandidates(modelId, region);
|
|
164
|
+
if (candidates.length === 0) {
|
|
165
|
+
throw error;
|
|
166
|
+
}
|
|
167
|
+
logger.warn("[Bedrock] Model requires a cross-region inference profile in this region; retrying with a prefixed id", { region, candidates: candidates.length });
|
|
168
|
+
for (const candidate of candidates) {
|
|
169
|
+
try {
|
|
170
|
+
const result = await send(candidate);
|
|
171
|
+
rememberResolvedModelId(modelId, region, candidate);
|
|
172
|
+
return result;
|
|
173
|
+
}
|
|
174
|
+
catch (retryError) {
|
|
175
|
+
if (!isInferenceProfileRequiredError(retryError)) {
|
|
176
|
+
// A different failure against the prefixed id — for example the
|
|
177
|
+
// account lacking access to that geography — is the more
|
|
178
|
+
// informative error, so surface it rather than the original.
|
|
179
|
+
throw retryError;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
throw error;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
//# sourceMappingURL=inferenceProfile.js.map
|
|
@@ -3,6 +3,7 @@ import path from "path";
|
|
|
3
3
|
import { createAnalytics } from "../../core/analytics.js";
|
|
4
4
|
import { BaseProvider } from "../../core/baseProvider.js";
|
|
5
5
|
import { DEFAULT_MAX_STEPS } from "../../core/constants.js";
|
|
6
|
+
import { withInferenceProfileFallback } from "./inferenceProfile.js";
|
|
6
7
|
import { AuthenticationError, ProviderError, RateLimitError, } from "../../types/index.js";
|
|
7
8
|
import { classifyProviderError } from "../../utils/errorClassifier.js";
|
|
8
9
|
import { isAbortError, withTimeout } from "../../utils/errorHandling.js";
|
|
@@ -370,8 +371,6 @@ export class AmazonBedrockProvider extends BaseProvider {
|
|
|
370
371
|
logger.info(` - Max tokens: ${commandInput.inferenceConfig?.maxTokens}`);
|
|
371
372
|
logger.info(` - Temperature: ${commandInput.inferenceConfig?.temperature}`);
|
|
372
373
|
logger.debug(`[AmazonBedrockProvider] Calling Bedrock with ${this.conversationHistory.length} messages and ${toolConfig?.tools?.length || 0} tools`);
|
|
373
|
-
// Create command and attempt API call
|
|
374
|
-
const command = new ConverseCommand(commandInput);
|
|
375
374
|
logger.debug("[Observability] Bedrock API request", {
|
|
376
375
|
model: commandInput.modelId,
|
|
377
376
|
region: region,
|
|
@@ -380,7 +379,15 @@ export class AmazonBedrockProvider extends BaseProvider {
|
|
|
380
379
|
maxTokens: commandInput.inferenceConfig?.maxTokens,
|
|
381
380
|
});
|
|
382
381
|
const apiCallStartTime = Date.now();
|
|
383
|
-
const response = await
|
|
382
|
+
const response = await withInferenceProfileFallback(commandInput.modelId ?? "",
|
|
383
|
+
// `this.region`, not the local `region` above: that one is
|
|
384
|
+
// best-effort for logging and stays "unknown" if the lookup
|
|
385
|
+
// throws. It must also match the id the streaming path keys its
|
|
386
|
+
// cache by, or a resolution found here is never reused there.
|
|
387
|
+
this.region, (effectiveModelId) => withTimeout(this.bedrockClient.send(new ConverseCommand({
|
|
388
|
+
...commandInput,
|
|
389
|
+
modelId: effectiveModelId,
|
|
390
|
+
})), 120_000, new Error("Bedrock API call timed out")));
|
|
384
391
|
const apiCallDuration = Date.now() - apiCallStartTime;
|
|
385
392
|
logger.debug("[Observability] Bedrock API response", {
|
|
386
393
|
model: commandInput.modelId,
|
|
@@ -1062,7 +1069,6 @@ export class AmazonBedrockProvider extends BaseProvider {
|
|
|
1062
1069
|
try {
|
|
1063
1070
|
logger.debug("[TRACE] streamingConversationLoop - testing first streaming call");
|
|
1064
1071
|
const commandInput = await this.prepareStreamCommand(options);
|
|
1065
|
-
const command = new ConverseStreamCommand(commandInput);
|
|
1066
1072
|
logger.debug("[Observability] Bedrock streaming API request", {
|
|
1067
1073
|
model: commandInput.modelId,
|
|
1068
1074
|
messageCount: commandInput.messages?.length || 0,
|
|
@@ -1073,7 +1079,10 @@ export class AmazonBedrockProvider extends BaseProvider {
|
|
|
1073
1079
|
"bedrock.tool_count": commandInput.toolConfig?.tools?.length || 0,
|
|
1074
1080
|
});
|
|
1075
1081
|
const streamStartTime = Date.now();
|
|
1076
|
-
const response = await withTimeout(this.bedrockClient.send(
|
|
1082
|
+
const response = await withInferenceProfileFallback(commandInput.modelId ?? "", this.region, (effectiveModelId) => withTimeout(this.bedrockClient.send(new ConverseStreamCommand({
|
|
1083
|
+
...commandInput,
|
|
1084
|
+
modelId: effectiveModelId,
|
|
1085
|
+
})), 120_000, new Error("Bedrock streaming API call timed out")));
|
|
1077
1086
|
logger.debug("[Observability] Bedrock streaming API connection established", {
|
|
1078
1087
|
model: commandInput.modelId,
|
|
1079
1088
|
durationMs: Date.now() - streamStartTime,
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bedrock cross-region inference profile resolution.
|
|
3
|
+
*
|
|
4
|
+
* Most current Bedrock models cannot be invoked by their bare model id. Each
|
|
5
|
+
* model card carries a Regional Availability table with three columns, and for
|
|
6
|
+
* a great many model/region pairs the In-Region column is "No" — Claude Opus
|
|
7
|
+
* 4.5 is No in every region AWS lists, Sonnet 4.6 is Yes only in eu-west-2.
|
|
8
|
+
* Those models are reachable only through a cross-region inference profile: the
|
|
9
|
+
* same id with a geography prefix (`us.`, `eu.`, `au.`, `jp.`, `apac.`) or the
|
|
10
|
+
* worldwide `global.` prefix.
|
|
11
|
+
*
|
|
12
|
+
* Sending the bare id where In-Region is No fails with a ValidationException
|
|
13
|
+
* telling you to use an inference profile. This module detects that specific
|
|
14
|
+
* failure and works out which prefixed id to retry with.
|
|
15
|
+
*
|
|
16
|
+
* ## Why prefixes are derived rather than tabulated
|
|
17
|
+
*
|
|
18
|
+
* The obvious implementation — a table of model to supported prefixes — is the
|
|
19
|
+
* one to avoid. The prefix set is per-model, not per-region: from the same
|
|
20
|
+
* caller region `ap-northeast-1`, Claude Opus 4.5 accepts only `global.`,
|
|
21
|
+
* Sonnet 4.5 also accepts `jp.`, and Haiku 4.5 works bare. A static table
|
|
22
|
+
* would need a row per model per region and would be wrong the day a model
|
|
23
|
+
* ships. AWS itself moved this data out of a central page and onto each
|
|
24
|
+
* model's detail page for the same reason.
|
|
25
|
+
*
|
|
26
|
+
* So resolution is empirical: on the specific error, try the geography that
|
|
27
|
+
* covers the caller's region, then `global.`, and remember what worked.
|
|
28
|
+
*
|
|
29
|
+
* `bedrock:ListInferenceProfiles` would give an authoritative answer, but it
|
|
30
|
+
* is a control-plane call needing a separate SDK client and an IAM permission
|
|
31
|
+
* a caller with plain inference access will not necessarily hold. Deriving
|
|
32
|
+
* candidates needs neither, and costs at most two extra attempts once per
|
|
33
|
+
* model/region pair.
|
|
34
|
+
*/
|
|
35
|
+
/**
|
|
36
|
+
* Does this error mean "this model needs an inference profile in this region"?
|
|
37
|
+
*
|
|
38
|
+
* Deliberately narrow. A ValidationException covers many unrelated causes —
|
|
39
|
+
* a malformed body, an unknown parameter, a model the account has no access to
|
|
40
|
+
* — and retrying those with a different model id would turn one clear failure
|
|
41
|
+
* into two confusing ones. Both the exception name and the distinctive phrase
|
|
42
|
+
* must match.
|
|
43
|
+
*
|
|
44
|
+
* The phrase is matched loosely enough to survive AWS rewording the sentence
|
|
45
|
+
* around it, and the retry is a no-op when nothing resolves, so a miss here
|
|
46
|
+
* costs nothing beyond the original error surfacing unchanged — which is the
|
|
47
|
+
* behaviour without this module at all.
|
|
48
|
+
*/
|
|
49
|
+
export declare function isInferenceProfileRequiredError(error: unknown): boolean;
|
|
50
|
+
/**
|
|
51
|
+
* Candidate ids to retry, most specific first: the caller's own geography,
|
|
52
|
+
* then worldwide.
|
|
53
|
+
*
|
|
54
|
+
* Returns an empty array when the id already carries a prefix — a prefixed id
|
|
55
|
+
* that still fails is a real error, not something to re-prefix.
|
|
56
|
+
*/
|
|
57
|
+
export declare function inferenceProfileCandidates(modelId: string, region: string): string[];
|
|
58
|
+
/** The id that previously worked for this model in this region, if any. */
|
|
59
|
+
export declare function getResolvedModelId(modelId: string, region: string): string | undefined;
|
|
60
|
+
export declare function rememberResolvedModelId(modelId: string, region: string, resolvedId: string): void;
|
|
61
|
+
/** Exposed so tests can start from a known state. */
|
|
62
|
+
export declare function clearResolvedModelIds(): void;
|
|
63
|
+
/**
|
|
64
|
+
* Run `send` against the given model id, falling back to inference-profile
|
|
65
|
+
* ids if — and only if — Bedrock says the bare id needs one.
|
|
66
|
+
*
|
|
67
|
+
* A previously resolved id is used straight away. Any error other than the
|
|
68
|
+
* inference-profile one propagates untouched on the first attempt, so this
|
|
69
|
+
* cannot mask an unrelated failure or silently change behaviour for a caller
|
|
70
|
+
* whose bare ids already work.
|
|
71
|
+
*/
|
|
72
|
+
export declare function withInferenceProfileFallback<T>(modelId: string, region: string, send: (effectiveModelId: string) => Promise<T>): Promise<T>;
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bedrock cross-region inference profile resolution.
|
|
3
|
+
*
|
|
4
|
+
* Most current Bedrock models cannot be invoked by their bare model id. Each
|
|
5
|
+
* model card carries a Regional Availability table with three columns, and for
|
|
6
|
+
* a great many model/region pairs the In-Region column is "No" — Claude Opus
|
|
7
|
+
* 4.5 is No in every region AWS lists, Sonnet 4.6 is Yes only in eu-west-2.
|
|
8
|
+
* Those models are reachable only through a cross-region inference profile: the
|
|
9
|
+
* same id with a geography prefix (`us.`, `eu.`, `au.`, `jp.`, `apac.`) or the
|
|
10
|
+
* worldwide `global.` prefix.
|
|
11
|
+
*
|
|
12
|
+
* Sending the bare id where In-Region is No fails with a ValidationException
|
|
13
|
+
* telling you to use an inference profile. This module detects that specific
|
|
14
|
+
* failure and works out which prefixed id to retry with.
|
|
15
|
+
*
|
|
16
|
+
* ## Why prefixes are derived rather than tabulated
|
|
17
|
+
*
|
|
18
|
+
* The obvious implementation — a table of model to supported prefixes — is the
|
|
19
|
+
* one to avoid. The prefix set is per-model, not per-region: from the same
|
|
20
|
+
* caller region `ap-northeast-1`, Claude Opus 4.5 accepts only `global.`,
|
|
21
|
+
* Sonnet 4.5 also accepts `jp.`, and Haiku 4.5 works bare. A static table
|
|
22
|
+
* would need a row per model per region and would be wrong the day a model
|
|
23
|
+
* ships. AWS itself moved this data out of a central page and onto each
|
|
24
|
+
* model's detail page for the same reason.
|
|
25
|
+
*
|
|
26
|
+
* So resolution is empirical: on the specific error, try the geography that
|
|
27
|
+
* covers the caller's region, then `global.`, and remember what worked.
|
|
28
|
+
*
|
|
29
|
+
* `bedrock:ListInferenceProfiles` would give an authoritative answer, but it
|
|
30
|
+
* is a control-plane call needing a separate SDK client and an IAM permission
|
|
31
|
+
* a caller with plain inference access will not necessarily hold. Deriving
|
|
32
|
+
* candidates needs neither, and costs at most two extra attempts once per
|
|
33
|
+
* model/region pair.
|
|
34
|
+
*/
|
|
35
|
+
import { logger } from "../../utils/logger.js";
|
|
36
|
+
/**
|
|
37
|
+
* Geography prefixes, in the order AWS documents them. `global.` is not a
|
|
38
|
+
* geography — it routes anywhere and is tried last, as the broadest option.
|
|
39
|
+
*/
|
|
40
|
+
const GLOBAL_PREFIX = "global";
|
|
41
|
+
/**
|
|
42
|
+
* Maps an AWS region to the geography whose inference profile covers it.
|
|
43
|
+
*
|
|
44
|
+
* Derived from the Geo inference tables on the model cards: the US geography
|
|
45
|
+
* covers `us-*` plus the Canadian regions, EU covers `eu-*` plus Israel, the
|
|
46
|
+
* Middle East and Africa, Japan covers the two Japanese regions, and Australia
|
|
47
|
+
* covers Sydney, Melbourne and New Zealand. Everything else in Asia Pacific
|
|
48
|
+
* falls under `apac`.
|
|
49
|
+
*/
|
|
50
|
+
function geoPrefixForRegion(region) {
|
|
51
|
+
if (/^(us|ca)-/.test(region)) {
|
|
52
|
+
return "us";
|
|
53
|
+
}
|
|
54
|
+
if (/^(eu|il|me|af)-/.test(region)) {
|
|
55
|
+
return "eu";
|
|
56
|
+
}
|
|
57
|
+
if (/^ap-northeast-[13]$/.test(region)) {
|
|
58
|
+
return "jp";
|
|
59
|
+
}
|
|
60
|
+
if (/^ap-southeast-[246]$/.test(region)) {
|
|
61
|
+
return "au";
|
|
62
|
+
}
|
|
63
|
+
if (/^ap-/.test(region)) {
|
|
64
|
+
return "apac";
|
|
65
|
+
}
|
|
66
|
+
return undefined;
|
|
67
|
+
}
|
|
68
|
+
/** True when `modelId` already carries a geography or global prefix. */
|
|
69
|
+
function hasProfilePrefix(modelId) {
|
|
70
|
+
return /^(us|eu|au|jp|apac|global)\./.test(modelId);
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Does this error mean "this model needs an inference profile in this region"?
|
|
74
|
+
*
|
|
75
|
+
* Deliberately narrow. A ValidationException covers many unrelated causes —
|
|
76
|
+
* a malformed body, an unknown parameter, a model the account has no access to
|
|
77
|
+
* — and retrying those with a different model id would turn one clear failure
|
|
78
|
+
* into two confusing ones. Both the exception name and the distinctive phrase
|
|
79
|
+
* must match.
|
|
80
|
+
*
|
|
81
|
+
* The phrase is matched loosely enough to survive AWS rewording the sentence
|
|
82
|
+
* around it, and the retry is a no-op when nothing resolves, so a miss here
|
|
83
|
+
* costs nothing beyond the original error surfacing unchanged — which is the
|
|
84
|
+
* behaviour without this module at all.
|
|
85
|
+
*/
|
|
86
|
+
export function isInferenceProfileRequiredError(error) {
|
|
87
|
+
if (typeof error !== "object" || error === null) {
|
|
88
|
+
return false;
|
|
89
|
+
}
|
|
90
|
+
const candidate = error;
|
|
91
|
+
const name = typeof candidate.name === "string" ? candidate.name : "";
|
|
92
|
+
const message = typeof candidate.message === "string" ? candidate.message : "";
|
|
93
|
+
if (!/ValidationException/i.test(name) &&
|
|
94
|
+
!/ValidationException/i.test(message)) {
|
|
95
|
+
return false;
|
|
96
|
+
}
|
|
97
|
+
return (/inference profile/i.test(message) && /on-demand throughput/i.test(message));
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Candidate ids to retry, most specific first: the caller's own geography,
|
|
101
|
+
* then worldwide.
|
|
102
|
+
*
|
|
103
|
+
* Returns an empty array when the id already carries a prefix — a prefixed id
|
|
104
|
+
* that still fails is a real error, not something to re-prefix.
|
|
105
|
+
*/
|
|
106
|
+
export function inferenceProfileCandidates(modelId, region) {
|
|
107
|
+
// An ARN already names a concrete model or inference-profile resource, and
|
|
108
|
+
// this provider accepts one as a model id (see `extractRegionFromArn`).
|
|
109
|
+
// Prefixing it yields a malformed identifier whose ValidationException would
|
|
110
|
+
// then replace the original, accurate error.
|
|
111
|
+
if (modelId.startsWith("arn:") || hasProfilePrefix(modelId)) {
|
|
112
|
+
return [];
|
|
113
|
+
}
|
|
114
|
+
const candidates = [];
|
|
115
|
+
const geo = geoPrefixForRegion(region);
|
|
116
|
+
if (geo) {
|
|
117
|
+
candidates.push(`${geo}.${modelId}`);
|
|
118
|
+
}
|
|
119
|
+
candidates.push(`${GLOBAL_PREFIX}.${modelId}`);
|
|
120
|
+
return candidates;
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Remembers which id actually worked, keyed by region and bare model id, so
|
|
124
|
+
* the probing happens once rather than on every call.
|
|
125
|
+
*/
|
|
126
|
+
const resolved = new Map();
|
|
127
|
+
function cacheKey(region, modelId) {
|
|
128
|
+
return `${region}::${modelId}`;
|
|
129
|
+
}
|
|
130
|
+
/** The id that previously worked for this model in this region, if any. */
|
|
131
|
+
export function getResolvedModelId(modelId, region) {
|
|
132
|
+
return resolved.get(cacheKey(region, modelId));
|
|
133
|
+
}
|
|
134
|
+
export function rememberResolvedModelId(modelId, region, resolvedId) {
|
|
135
|
+
resolved.set(cacheKey(region, modelId), resolvedId);
|
|
136
|
+
logger.debug("[Bedrock] Cached inference-profile resolution for subsequent calls", { region });
|
|
137
|
+
}
|
|
138
|
+
/** Exposed so tests can start from a known state. */
|
|
139
|
+
export function clearResolvedModelIds() {
|
|
140
|
+
resolved.clear();
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Run `send` against the given model id, falling back to inference-profile
|
|
144
|
+
* ids if — and only if — Bedrock says the bare id needs one.
|
|
145
|
+
*
|
|
146
|
+
* A previously resolved id is used straight away. Any error other than the
|
|
147
|
+
* inference-profile one propagates untouched on the first attempt, so this
|
|
148
|
+
* cannot mask an unrelated failure or silently change behaviour for a caller
|
|
149
|
+
* whose bare ids already work.
|
|
150
|
+
*/
|
|
151
|
+
export async function withInferenceProfileFallback(modelId, region, send) {
|
|
152
|
+
const cached = getResolvedModelId(modelId, region);
|
|
153
|
+
if (cached && cached !== modelId) {
|
|
154
|
+
return send(cached);
|
|
155
|
+
}
|
|
156
|
+
try {
|
|
157
|
+
return await send(modelId);
|
|
158
|
+
}
|
|
159
|
+
catch (error) {
|
|
160
|
+
if (!isInferenceProfileRequiredError(error)) {
|
|
161
|
+
throw error;
|
|
162
|
+
}
|
|
163
|
+
const candidates = inferenceProfileCandidates(modelId, region);
|
|
164
|
+
if (candidates.length === 0) {
|
|
165
|
+
throw error;
|
|
166
|
+
}
|
|
167
|
+
logger.warn("[Bedrock] Model requires a cross-region inference profile in this region; retrying with a prefixed id", { region, candidates: candidates.length });
|
|
168
|
+
for (const candidate of candidates) {
|
|
169
|
+
try {
|
|
170
|
+
const result = await send(candidate);
|
|
171
|
+
rememberResolvedModelId(modelId, region, candidate);
|
|
172
|
+
return result;
|
|
173
|
+
}
|
|
174
|
+
catch (retryError) {
|
|
175
|
+
if (!isInferenceProfileRequiredError(retryError)) {
|
|
176
|
+
// A different failure against the prefixed id — for example the
|
|
177
|
+
// account lacking access to that geography — is the more
|
|
178
|
+
// informative error, so surface it rather than the original.
|
|
179
|
+
throw retryError;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
throw error;
|
|
184
|
+
}
|
|
185
|
+
}
|