@juspay/neurolink 12.56.0 → 12.56.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +3 -3
- package/dist/browser/neurolink.min.js +409 -409
- package/dist/neurolink.d.ts +12 -8
- package/dist/neurolink.js +89 -25
- package/dist/types/index.d.ts +1 -0
- package/dist/types/index.js +1 -0
- package/dist/types/ollamaDiagnostics.d.ts +31 -0
- package/dist/types/ollamaDiagnostics.js +1 -0
- package/dist/utils/ollamaDiagnostics.d.ts +69 -0
- package/dist/utils/ollamaDiagnostics.js +192 -0
- package/dist/utils/providerHealth.d.ts +33 -19
- package/dist/utils/providerHealth.js +224 -85
- package/package.json +3 -1
|
@@ -1,7 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Provider Health Checking System
|
|
3
|
-
* Prevents 500 errors by validating provider availability and configuration
|
|
4
|
-
*/
|
|
5
1
|
import { AIProviderName } from "../constants/enums.js";
|
|
6
2
|
import type { ProviderHealthCheckOptions, ProviderHealthStatusOptions } from "../types/index.js";
|
|
7
3
|
export declare class ProviderHealthChecker {
|
|
@@ -9,20 +5,16 @@ export declare class ProviderHealthChecker {
|
|
|
9
5
|
private static readonly DEFAULT_TIMEOUT;
|
|
10
6
|
private static readonly DEFAULT_CACHE_AGE;
|
|
11
7
|
/**
|
|
12
|
-
* Fixed backoff
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
* `maxCacheAge` for its own health-status cache). Evicting the breaker
|
|
17
|
-
* entry using whichever caller happens to read it next would let a
|
|
18
|
-
* short-TTL caller silently erase a backoff another caller is relying on
|
|
19
|
-
* — see `checkProviderHealth`. This constant governs breaker expiry alone
|
|
20
|
-
* and is intentionally independent of `maxCacheAge`/`DEFAULT_CACHE_AGE`.
|
|
8
|
+
* Fixed backoff for the existing non-Ollama provider breaker. Ollama uses
|
|
9
|
+
* the same fixed window in ollamaDiagnostics, shared by every native caller.
|
|
10
|
+
* Neither window depends on a caller's maxCacheAge: expiring a health cache
|
|
11
|
+
* must not silently erase a transport backoff owned by another caller.
|
|
21
12
|
*/
|
|
22
13
|
private static readonly CIRCUIT_BREAKER_RESET_MS;
|
|
23
14
|
private static readonly CONSECUTIVE_FAILURE_THRESHOLD;
|
|
24
15
|
/**
|
|
25
|
-
*
|
|
16
|
+
* Non-Ollama circuit breaker state, keyed by provider. Ollama's separate
|
|
17
|
+
* native policy captures the actual endpoint and records aggregate outcomes.
|
|
26
18
|
*
|
|
27
19
|
* `count` is a count of consecutive failed CONNECTIVITY PROBES (plus the
|
|
28
20
|
* rare case of the check itself throwing) — never of "the provider is
|
|
@@ -56,11 +48,14 @@ export declare class ProviderHealthChecker {
|
|
|
56
48
|
*/
|
|
57
49
|
private static readonly MAX_CONCURRENT_HEALTH_CHECKS;
|
|
58
50
|
/**
|
|
59
|
-
*
|
|
51
|
+
* Preserve the validated threshold's original first-use initialization.
|
|
60
52
|
*/
|
|
61
53
|
private static getValidatedFailureThreshold;
|
|
62
54
|
/**
|
|
63
|
-
*
|
|
55
|
+
* Check configuration and optional connectivity/model availability.
|
|
56
|
+
* Ollama captures its endpoint/model once and records only probes that run;
|
|
57
|
+
* a cache hit or missing configured model does not invent a transport failure.
|
|
58
|
+
* Its cached result must match the current diagnostic revision/generation.
|
|
64
59
|
*/
|
|
65
60
|
static checkProviderHealth(providerName: AIProviderName, options?: ProviderHealthCheckOptions): Promise<ProviderHealthStatusOptions>;
|
|
66
61
|
/**
|
|
@@ -189,18 +184,33 @@ export declare class ProviderHealthChecker {
|
|
|
189
184
|
private static getLiteLLMModelsUrl;
|
|
190
185
|
private static getConfiguredLiteLLMModel;
|
|
191
186
|
private static getOllamaBaseUrl;
|
|
187
|
+
/** Resolve the shared native tags URL rather than the compatible models URL. */
|
|
192
188
|
private static getOllamaTagsUrl;
|
|
193
189
|
private static getConfiguredOllamaModel;
|
|
194
190
|
private static fetchJsonWithTimeout;
|
|
195
191
|
private static normalizeModelList;
|
|
196
192
|
private static hasRequestedModel;
|
|
193
|
+
/**
|
|
194
|
+
* Fetch native tags from the captured endpoint and normalize its models.
|
|
195
|
+
* A malformed envelope rejects; a valid empty list remains an empty result.
|
|
196
|
+
*/
|
|
197
197
|
private static getOllamaAvailableModels;
|
|
198
198
|
private static getLiteLLMAvailableModels;
|
|
199
|
+
/**
|
|
200
|
+
* Check a requested model against native tags while keeping diagnostic outcome
|
|
201
|
+
* separate from model availability. An answered valid list is diagnostic
|
|
202
|
+
* success even if that model is absent; malformed JSON/envelopes are protocol
|
|
203
|
+
* failures. A caller recording an aggregate health probe disables recording
|
|
204
|
+
* here so its admitted permit is completed only once at the outer boundary.
|
|
205
|
+
*/
|
|
199
206
|
private static checkOllamaAvailability;
|
|
200
207
|
private static checkLiteLLMAvailability;
|
|
201
208
|
private static checkLiteLLMConfig;
|
|
202
209
|
/**
|
|
203
|
-
*
|
|
210
|
+
* Report Ollama configuration and model availability using the captured context.
|
|
211
|
+
* Backoff can suppress native work; a missing installed model is a configuration
|
|
212
|
+
* issue after a successful probe. Return actual probe activity/failure for the
|
|
213
|
+
* enclosing health check to record, without completing a second policy permit.
|
|
204
214
|
*/
|
|
205
215
|
private static checkOllamaConfig;
|
|
206
216
|
/**
|
|
@@ -208,7 +218,9 @@ export declare class ProviderHealthChecker {
|
|
|
208
218
|
*/
|
|
209
219
|
private static getCommonModelsForProvider;
|
|
210
220
|
/**
|
|
211
|
-
*
|
|
221
|
+
* Return health within the caller's TTL and, for Ollama, the captured identity.
|
|
222
|
+
* Ollama revision/generation changes invalidate cached status; reading the
|
|
223
|
+
* snapshot may apply timed expiry but never records a probe outcome.
|
|
212
224
|
*/
|
|
213
225
|
private static getCachedHealth;
|
|
214
226
|
/**
|
|
@@ -267,7 +279,9 @@ export declare class ProviderHealthChecker {
|
|
|
267
279
|
*/
|
|
268
280
|
static initializeBackgroundHealthChecks(): void;
|
|
269
281
|
/**
|
|
270
|
-
* Clear health
|
|
282
|
+
* Clear cached health and failure state for one provider or every provider.
|
|
283
|
+
* Clearing Ollama removes all endpoint/model cache entries and explicitly
|
|
284
|
+
* advances diagnostic generation so older in-flight permits cannot commit.
|
|
271
285
|
*/
|
|
272
286
|
static clearHealthCache(providerName?: AIProviderName): void;
|
|
273
287
|
static checkFallbackProviderAvailability(providerName: string, model: string): Promise<{
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Provider
|
|
3
|
-
*
|
|
2
|
+
* Provider health checking with captured native diagnostics for Ollama.
|
|
3
|
+
* Configuration/model availability and transport failures remain separate.
|
|
4
4
|
*/
|
|
5
|
+
import { beginOllamaDiagnostic, captureOllamaDiagnosticContext, clearOllamaDiagnostics, completeOllamaDiagnostic, getOllamaDiagnosticErrorMessage, getOllamaDiagnosticSnapshot, getOllamaHealthCacheKey, getValidatedProviderFailureThreshold, OllamaProtocolError, readOllamaTagsModels, resolveOllamaNativeTagsUrl, } from "./ollamaDiagnostics.js";
|
|
5
6
|
import { logger } from "./logger.js";
|
|
6
7
|
import { AIProviderName, OpenAIModels, GoogleAIModels, AnthropicModels, BedrockModels, } from "../constants/enums.js";
|
|
7
8
|
import { PROJECT_ID_FORMAT, satisfiesFallbacks } from "./providerConfig.js";
|
|
@@ -15,20 +16,16 @@ export class ProviderHealthChecker {
|
|
|
15
16
|
static DEFAULT_TIMEOUT = 5000; // 5 seconds
|
|
16
17
|
static DEFAULT_CACHE_AGE = 300000; // 5 minutes
|
|
17
18
|
/**
|
|
18
|
-
* Fixed backoff
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
* `maxCacheAge` for its own health-status cache). Evicting the breaker
|
|
23
|
-
* entry using whichever caller happens to read it next would let a
|
|
24
|
-
* short-TTL caller silently erase a backoff another caller is relying on
|
|
25
|
-
* — see `checkProviderHealth`. This constant governs breaker expiry alone
|
|
26
|
-
* and is intentionally independent of `maxCacheAge`/`DEFAULT_CACHE_AGE`.
|
|
19
|
+
* Fixed backoff for the existing non-Ollama provider breaker. Ollama uses
|
|
20
|
+
* the same fixed window in ollamaDiagnostics, shared by every native caller.
|
|
21
|
+
* Neither window depends on a caller's maxCacheAge: expiring a health cache
|
|
22
|
+
* must not silently erase a transport backoff owned by another caller.
|
|
27
23
|
*/
|
|
28
24
|
static CIRCUIT_BREAKER_RESET_MS = 300000; // 5 minutes
|
|
29
25
|
static CONSECUTIVE_FAILURE_THRESHOLD = ProviderHealthChecker.getValidatedFailureThreshold();
|
|
30
26
|
/**
|
|
31
|
-
*
|
|
27
|
+
* Non-Ollama circuit breaker state, keyed by provider. Ollama's separate
|
|
28
|
+
* native policy captures the actual endpoint and records aggregate outcomes.
|
|
32
29
|
*
|
|
33
30
|
* `count` is a count of consecutive failed CONNECTIVITY PROBES (plus the
|
|
34
31
|
* rare case of the check itself throwing) — never of "the provider is
|
|
@@ -62,28 +59,35 @@ export class ProviderHealthChecker {
|
|
|
62
59
|
*/
|
|
63
60
|
static MAX_CONCURRENT_HEALTH_CHECKS = 8;
|
|
64
61
|
/**
|
|
65
|
-
*
|
|
62
|
+
* Preserve the validated threshold's original first-use initialization.
|
|
66
63
|
*/
|
|
67
64
|
static getValidatedFailureThreshold() {
|
|
68
|
-
|
|
69
|
-
if (!envValue) {
|
|
70
|
-
return 3; // default
|
|
71
|
-
}
|
|
72
|
-
const parsed = Number(envValue);
|
|
73
|
-
if (isNaN(parsed) || parsed <= 0 || parsed > 10) {
|
|
74
|
-
logger.warn(`Invalid PROVIDER_FAILURE_THRESHOLD: ${envValue} (must be between 1 and 10), using default: 3`);
|
|
75
|
-
return 3;
|
|
76
|
-
}
|
|
77
|
-
return parsed;
|
|
65
|
+
return getValidatedProviderFailureThreshold();
|
|
78
66
|
}
|
|
79
67
|
/**
|
|
80
|
-
*
|
|
68
|
+
* Check configuration and optional connectivity/model availability.
|
|
69
|
+
* Ollama captures its endpoint/model once and records only probes that run;
|
|
70
|
+
* a cache hit or missing configured model does not invent a transport failure.
|
|
71
|
+
* Its cached result must match the current diagnostic revision/generation.
|
|
81
72
|
*/
|
|
82
73
|
static async checkProviderHealth(providerName, options = {}) {
|
|
83
74
|
const { timeout = this.DEFAULT_TIMEOUT, includeConnectivityTest = false, includeModelValidation = false, cacheResults = true, maxCacheAge = this.DEFAULT_CACHE_AGE, } = options;
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
75
|
+
const isOllama = providerName === AIProviderName.OLLAMA;
|
|
76
|
+
let ollamaContext;
|
|
77
|
+
if (isOllama) {
|
|
78
|
+
try {
|
|
79
|
+
ollamaContext = captureOllamaDiagnosticContext();
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
// The local configuration path below reports the invalid URL without
|
|
83
|
+
// fabricating an outbound transport failure or reusing old cached health.
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
let ollamaProbeRan = false;
|
|
87
|
+
let ollamaProbeFailed = false;
|
|
88
|
+
// Cache hits remain observations: they neither begin nor complete a probe.
|
|
89
|
+
if (cacheResults && (!isOllama || ollamaContext)) {
|
|
90
|
+
const cached = this.getCachedHealth(providerName, maxCacheAge, ollamaContext);
|
|
87
91
|
if (cached) {
|
|
88
92
|
logger.debug(`Using cached health status for ${providerName}`);
|
|
89
93
|
return cached;
|
|
@@ -112,24 +116,26 @@ export class ProviderHealthChecker {
|
|
|
112
116
|
// down, so a dead local proxy still gets backed off instead of eating a
|
|
113
117
|
// full timeout on every call forever.
|
|
114
118
|
//
|
|
115
|
-
//
|
|
116
|
-
//
|
|
117
|
-
//
|
|
118
|
-
// breaker's whole effect is to suppress the probe that would disprove
|
|
119
|
-
// it. This is deliberately the fixed CIRCUIT_BREAKER_RESET_MS window,
|
|
120
|
-
// not this call's own `maxCacheAge` — `consecutiveFailures` is a single
|
|
121
|
-
// shared map keyed only by provider name, so reusing whichever caller
|
|
122
|
-
// happens to invoke `checkProviderHealth` next (some call sites pass a
|
|
123
|
-
// much shorter `maxCacheAge` for their own unrelated caching needs)
|
|
124
|
-
// would let that caller silently evict a backoff a different caller is
|
|
125
|
-
// relying on.
|
|
119
|
+
// Ollama consults its captured endpoint policy; other providers retain
|
|
120
|
+
// their existing provider-keyed map. Both expire at the fixed reset window,
|
|
121
|
+
// independently of this caller's health-cache TTL.
|
|
126
122
|
const breaker = this.consecutiveFailures.get(providerName);
|
|
127
|
-
if (
|
|
123
|
+
if (!isOllama &&
|
|
124
|
+
breaker !== undefined &&
|
|
128
125
|
Date.now() - breaker.lastFailureAt >= this.CIRCUIT_BREAKER_RESET_MS) {
|
|
129
126
|
this.consecutiveFailures.delete(providerName);
|
|
130
127
|
}
|
|
131
|
-
const
|
|
132
|
-
|
|
128
|
+
const ollamaGate = ollamaContext
|
|
129
|
+
? beginOllamaDiagnostic(ollamaContext)
|
|
130
|
+
: undefined;
|
|
131
|
+
const failureCount = isOllama
|
|
132
|
+
? ollamaContext
|
|
133
|
+
? getOllamaDiagnosticSnapshot(ollamaContext).count
|
|
134
|
+
: 0
|
|
135
|
+
: (this.consecutiveFailures.get(providerName)?.count ?? 0);
|
|
136
|
+
const blacklisted = isOllama
|
|
137
|
+
? ollamaGate?.kind === "suppressed"
|
|
138
|
+
: failureCount >= this.CONSECUTIVE_FAILURE_THRESHOLD;
|
|
133
139
|
if (blacklisted) {
|
|
134
140
|
logger.warn(`Provider ${providerName} blacklisted due to consecutive failures`, { failureCount });
|
|
135
141
|
}
|
|
@@ -147,14 +153,16 @@ export class ProviderHealthChecker {
|
|
|
147
153
|
// 1. Check environment configuration. For LiteLLM/Ollama this
|
|
148
154
|
// includes a breaker-governed runtime probe — see the comment on
|
|
149
155
|
// `blacklisted` above.
|
|
150
|
-
const runtimeProbe = await this.checkEnvironmentConfiguration(providerName, healthStatus, timeout, !blacklisted);
|
|
156
|
+
const runtimeProbe = await this.checkEnvironmentConfiguration(providerName, healthStatus, timeout, !blacklisted, ollamaContext);
|
|
157
|
+
ollamaProbeRan = runtimeProbe.ran;
|
|
158
|
+
ollamaProbeFailed = runtimeProbe.failed;
|
|
151
159
|
// 2. Check API key validity (basic format validation)
|
|
152
160
|
await this.checkApiKeyValidity(providerName, healthStatus);
|
|
153
161
|
// 3. Optional: Connectivity test. Also governed by the breaker.
|
|
154
162
|
const issuesBeforeProbe = healthStatus.configurationIssues.length;
|
|
155
163
|
const probed = includeConnectivityTest && !blacklisted;
|
|
156
164
|
if (probed) {
|
|
157
|
-
await this.checkConnectivity(providerName, healthStatus, timeout);
|
|
165
|
+
await this.checkConnectivity(providerName, healthStatus, timeout, ollamaContext);
|
|
158
166
|
}
|
|
159
167
|
const probeFailed = probed && healthStatus.configurationIssues.length > issuesBeforeProbe;
|
|
160
168
|
// Fold the step-1 runtime probe (if this provider has one) together
|
|
@@ -164,6 +172,8 @@ export class ProviderHealthChecker {
|
|
|
164
172
|
// requests).
|
|
165
173
|
const anyProbeRan = probed || runtimeProbe.ran;
|
|
166
174
|
const anyProbeFailed = probeFailed || runtimeProbe.failed;
|
|
175
|
+
ollamaProbeRan = anyProbeRan;
|
|
176
|
+
ollamaProbeFailed = anyProbeFailed;
|
|
167
177
|
// 4. Optional: Model validation
|
|
168
178
|
if (includeModelValidation) {
|
|
169
179
|
await this.checkModelAvailability(providerName, healthStatus);
|
|
@@ -182,7 +192,7 @@ export class ProviderHealthChecker {
|
|
|
182
192
|
healthStatus.configurationIssues.length === 0;
|
|
183
193
|
healthStatus.responseTime = Date.now() - startTime;
|
|
184
194
|
// Cache results
|
|
185
|
-
if (cacheResults) {
|
|
195
|
+
if (cacheResults && !isOllama) {
|
|
186
196
|
this.healthCache.set(providerName, {
|
|
187
197
|
status: healthStatus,
|
|
188
198
|
timestamp: Date.now(),
|
|
@@ -191,14 +201,38 @@ export class ProviderHealthChecker {
|
|
|
191
201
|
// Only a probe that actually ran moves the breaker. A check that
|
|
192
202
|
// issued no request is not evidence either way: it neither proves the
|
|
193
203
|
// provider is failing nor that it has recovered.
|
|
194
|
-
if (
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
204
|
+
if (isOllama) {
|
|
205
|
+
if (ollamaGate?.kind === "allowed") {
|
|
206
|
+
completeOllamaDiagnostic(ollamaGate, anyProbeFailed
|
|
207
|
+
? "transport-failure"
|
|
208
|
+
: anyProbeRan
|
|
209
|
+
? "success"
|
|
210
|
+
: "not-run");
|
|
211
|
+
}
|
|
212
|
+
if (cacheResults && ollamaContext) {
|
|
213
|
+
const snapshot = getOllamaDiagnosticSnapshot(ollamaContext);
|
|
214
|
+
if (ollamaGate && snapshot.generation === ollamaGate.generation) {
|
|
215
|
+
// Bind the FINAL recorded revision, never the pre-request revision.
|
|
216
|
+
// Clearing also invalidates a suppressed call's pending cache write.
|
|
217
|
+
this.healthCache.set(getOllamaHealthCacheKey(ollamaContext), {
|
|
218
|
+
status: healthStatus,
|
|
219
|
+
timestamp: Date.now(),
|
|
220
|
+
ollamaRevision: snapshot.revision,
|
|
221
|
+
ollamaGeneration: snapshot.generation,
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
}
|
|
199
225
|
}
|
|
200
|
-
else
|
|
201
|
-
|
|
226
|
+
else {
|
|
227
|
+
if (anyProbeFailed) {
|
|
228
|
+
this.consecutiveFailures.set(providerName, {
|
|
229
|
+
count: failureCount + 1,
|
|
230
|
+
lastFailureAt: Date.now(),
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
else if (anyProbeRan) {
|
|
234
|
+
this.consecutiveFailures.delete(providerName);
|
|
235
|
+
}
|
|
202
236
|
}
|
|
203
237
|
logger.debug(`Health check completed for ${providerName}`, {
|
|
204
238
|
isHealthy: healthStatus.isHealthy,
|
|
@@ -207,19 +241,38 @@ export class ProviderHealthChecker {
|
|
|
207
241
|
});
|
|
208
242
|
}
|
|
209
243
|
catch (error) {
|
|
210
|
-
const errorMessage =
|
|
244
|
+
const errorMessage = isOllama
|
|
245
|
+
? getOllamaDiagnosticErrorMessage(error)
|
|
246
|
+
: error instanceof Error
|
|
247
|
+
? error.message
|
|
248
|
+
: String(error);
|
|
211
249
|
healthStatus.error = errorMessage;
|
|
212
250
|
healthStatus.configurationIssues.push(`Health check failed: ${errorMessage}`);
|
|
213
251
|
healthStatus.responseTime = Date.now() - startTime;
|
|
214
|
-
//
|
|
215
|
-
//
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
252
|
+
// Keep legacy error accounting for other providers. Ollama records only
|
|
253
|
+
// probes that actually ran, so a local exception cannot invent an outage.
|
|
254
|
+
if (isOllama) {
|
|
255
|
+
if (ollamaGate?.kind === "allowed") {
|
|
256
|
+
completeOllamaDiagnostic(ollamaGate, ollamaProbeFailed
|
|
257
|
+
? "transport-failure"
|
|
258
|
+
: ollamaProbeRan
|
|
259
|
+
? "success"
|
|
260
|
+
: "not-run");
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
else {
|
|
264
|
+
this.consecutiveFailures.set(providerName, {
|
|
265
|
+
count: failureCount + 1,
|
|
266
|
+
lastFailureAt: Date.now(),
|
|
267
|
+
});
|
|
268
|
+
}
|
|
220
269
|
logger.warn(`Health check failed for ${providerName}`, {
|
|
221
270
|
error: errorMessage,
|
|
222
|
-
consecutiveFailures:
|
|
271
|
+
consecutiveFailures: isOllama
|
|
272
|
+
? ollamaContext
|
|
273
|
+
? getOllamaDiagnosticSnapshot(ollamaContext).count
|
|
274
|
+
: 0
|
|
275
|
+
: failureCount + 1,
|
|
223
276
|
});
|
|
224
277
|
}
|
|
225
278
|
return healthStatus;
|
|
@@ -227,7 +280,7 @@ export class ProviderHealthChecker {
|
|
|
227
280
|
/**
|
|
228
281
|
* Check environment configuration for a provider
|
|
229
282
|
*/
|
|
230
|
-
static async checkEnvironmentConfiguration(providerName, healthStatus, timeout, allowRuntimeProbe) {
|
|
283
|
+
static async checkEnvironmentConfiguration(providerName, healthStatus, timeout, allowRuntimeProbe, ollamaContext) {
|
|
231
284
|
const requiredEnvVars = this.getRequiredEnvironmentVariables(providerName);
|
|
232
285
|
logger.debug(`[ProviderHealthChecker] Checking environment configuration for ${providerName}`, {
|
|
233
286
|
requiredEnvVars,
|
|
@@ -258,7 +311,7 @@ export class ProviderHealthChecker {
|
|
|
258
311
|
healthStatus.recommendations.push(`Set the following environment variables: ${missingVars.join(", ")}`);
|
|
259
312
|
}
|
|
260
313
|
// Provider-specific configuration checks
|
|
261
|
-
return this.checkProviderSpecificConfig(providerName, healthStatus, timeout, allowRuntimeProbe);
|
|
314
|
+
return this.checkProviderSpecificConfig(providerName, healthStatus, timeout, allowRuntimeProbe, ollamaContext);
|
|
262
315
|
}
|
|
263
316
|
/**
|
|
264
317
|
* Check API key validity (format validation)
|
|
@@ -341,8 +394,10 @@ export class ProviderHealthChecker {
|
|
|
341
394
|
/**
|
|
342
395
|
* Check connectivity to provider endpoints
|
|
343
396
|
*/
|
|
344
|
-
static async checkConnectivity(providerName, healthStatus, timeout) {
|
|
345
|
-
const endpoint =
|
|
397
|
+
static async checkConnectivity(providerName, healthStatus, timeout, ollamaContext) {
|
|
398
|
+
const endpoint = providerName === AIProviderName.OLLAMA
|
|
399
|
+
? (ollamaContext?.url ?? null)
|
|
400
|
+
: this.getProviderHealthEndpoint(providerName);
|
|
346
401
|
if (!endpoint) {
|
|
347
402
|
healthStatus.warning = "No connectivity test available for this provider";
|
|
348
403
|
return;
|
|
@@ -558,7 +613,7 @@ export class ProviderHealthChecker {
|
|
|
558
613
|
/**
|
|
559
614
|
* Provider-specific configuration checks
|
|
560
615
|
*/
|
|
561
|
-
static async checkProviderSpecificConfig(providerName, healthStatus, timeout, allowRuntimeProbe) {
|
|
616
|
+
static async checkProviderSpecificConfig(providerName, healthStatus, timeout, allowRuntimeProbe, ollamaContext) {
|
|
562
617
|
switch (providerName) {
|
|
563
618
|
case AIProviderName.VERTEX:
|
|
564
619
|
await this.checkVertexAIConfig(healthStatus);
|
|
@@ -572,7 +627,7 @@ export class ProviderHealthChecker {
|
|
|
572
627
|
case AIProviderName.LITELLM:
|
|
573
628
|
return this.checkLiteLLMConfig(healthStatus, timeout, allowRuntimeProbe);
|
|
574
629
|
case AIProviderName.OLLAMA:
|
|
575
|
-
return this.checkOllamaConfig(healthStatus, timeout, allowRuntimeProbe);
|
|
630
|
+
return this.checkOllamaConfig(healthStatus, timeout, allowRuntimeProbe, ollamaContext);
|
|
576
631
|
}
|
|
577
632
|
// Every other provider's config check above is local (env vars, file
|
|
578
633
|
// existence) — never a network request, so there is no probe to report.
|
|
@@ -785,8 +840,9 @@ export class ProviderHealthChecker {
|
|
|
785
840
|
process.env.OLLAMA_API_BASE ||
|
|
786
841
|
"http://localhost:11434");
|
|
787
842
|
}
|
|
843
|
+
/** Resolve the shared native tags URL rather than the compatible models URL. */
|
|
788
844
|
static getOllamaTagsUrl() {
|
|
789
|
-
return
|
|
845
|
+
return resolveOllamaNativeTagsUrl();
|
|
790
846
|
}
|
|
791
847
|
static getConfiguredOllamaModel() {
|
|
792
848
|
return process.env.OLLAMA_MODEL || DEFAULT_OLLAMA_MODEL;
|
|
@@ -840,11 +896,13 @@ export class ProviderHealthChecker {
|
|
|
840
896
|
(model.startsWith(`${normalizedRequestedModel}:`) ||
|
|
841
897
|
model.startsWith(`${normalizedRequestedModel}@`))));
|
|
842
898
|
}
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
899
|
+
/**
|
|
900
|
+
* Fetch native tags from the captured endpoint and normalize its models.
|
|
901
|
+
* A malformed envelope rejects; a valid empty list remains an empty result.
|
|
902
|
+
*/
|
|
903
|
+
static async getOllamaAvailableModels(timeout = 2000, endpoint = this.getOllamaTagsUrl()) {
|
|
904
|
+
const payload = await this.fetchJsonWithTimeout(endpoint, { timeout });
|
|
905
|
+
return this.normalizeModelList(readOllamaTagsModels(payload));
|
|
848
906
|
}
|
|
849
907
|
static async getLiteLLMAvailableModels(timeout = 2000) {
|
|
850
908
|
const payload = (await this.fetchJsonWithTimeout(this.getLiteLLMModelsUrl(), {
|
|
@@ -856,25 +914,69 @@ export class ProviderHealthChecker {
|
|
|
856
914
|
}));
|
|
857
915
|
return this.normalizeModelList(payload.data ?? []);
|
|
858
916
|
}
|
|
917
|
+
/**
|
|
918
|
+
* Check a requested model against native tags while keeping diagnostic outcome
|
|
919
|
+
* separate from model availability. An answered valid list is diagnostic
|
|
920
|
+
* success even if that model is absent; malformed JSON/envelopes are protocol
|
|
921
|
+
* failures. A caller recording an aggregate health probe disables recording
|
|
922
|
+
* here so its admitted permit is completed only once at the outer boundary.
|
|
923
|
+
*/
|
|
859
924
|
static async checkOllamaAvailability(options) {
|
|
925
|
+
let context;
|
|
926
|
+
try {
|
|
927
|
+
context =
|
|
928
|
+
options.context ?? captureOllamaDiagnosticContext(options.model);
|
|
929
|
+
}
|
|
930
|
+
catch (error) {
|
|
931
|
+
return {
|
|
932
|
+
available: false,
|
|
933
|
+
reason: getOllamaDiagnosticErrorMessage(error),
|
|
934
|
+
models: [],
|
|
935
|
+
diagnosticOutcome: "configuration-failure",
|
|
936
|
+
};
|
|
937
|
+
}
|
|
938
|
+
const gate = options.recordDiagnostic === false
|
|
939
|
+
? undefined
|
|
940
|
+
: beginOllamaDiagnostic(context);
|
|
941
|
+
if (gate?.kind === "suppressed") {
|
|
942
|
+
return {
|
|
943
|
+
available: false,
|
|
944
|
+
reason: gate.reason,
|
|
945
|
+
models: [],
|
|
946
|
+
diagnosticOutcome: "suppressed",
|
|
947
|
+
};
|
|
948
|
+
}
|
|
949
|
+
let outcome;
|
|
860
950
|
try {
|
|
861
|
-
const models = await this.getOllamaAvailableModels(options.timeout);
|
|
951
|
+
const models = await this.getOllamaAvailableModels(options.timeout, context.url);
|
|
952
|
+
outcome = "success";
|
|
862
953
|
if (!this.hasRequestedModel(models, options.model)) {
|
|
863
954
|
return {
|
|
864
955
|
available: false,
|
|
865
956
|
reason: `Configured Ollama model '${options.model}' is not installed`,
|
|
866
957
|
models,
|
|
958
|
+
diagnosticOutcome: outcome,
|
|
867
959
|
};
|
|
868
960
|
}
|
|
869
|
-
return { available: true, models };
|
|
961
|
+
return { available: true, models, diagnosticOutcome: outcome };
|
|
870
962
|
}
|
|
871
963
|
catch (error) {
|
|
964
|
+
outcome =
|
|
965
|
+
error instanceof OllamaProtocolError || error instanceof SyntaxError
|
|
966
|
+
? "protocol-failure"
|
|
967
|
+
: "transport-failure";
|
|
872
968
|
return {
|
|
873
969
|
available: false,
|
|
874
|
-
reason:
|
|
970
|
+
reason: getOllamaDiagnosticErrorMessage(error),
|
|
875
971
|
models: [],
|
|
972
|
+
diagnosticOutcome: outcome,
|
|
876
973
|
};
|
|
877
974
|
}
|
|
975
|
+
finally {
|
|
976
|
+
if (gate?.kind === "allowed") {
|
|
977
|
+
completeOllamaDiagnostic(gate, outcome ?? "not-run");
|
|
978
|
+
}
|
|
979
|
+
}
|
|
878
980
|
}
|
|
879
981
|
static async checkLiteLLMAvailability(options) {
|
|
880
982
|
try {
|
|
@@ -958,10 +1060,13 @@ export class ProviderHealthChecker {
|
|
|
958
1060
|
return { ran: true, failed: false };
|
|
959
1061
|
}
|
|
960
1062
|
/**
|
|
961
|
-
*
|
|
1063
|
+
* Report Ollama configuration and model availability using the captured context.
|
|
1064
|
+
* Backoff can suppress native work; a missing installed model is a configuration
|
|
1065
|
+
* issue after a successful probe. Return actual probe activity/failure for the
|
|
1066
|
+
* enclosing health check to record, without completing a second policy permit.
|
|
962
1067
|
*/
|
|
963
|
-
static async checkOllamaConfig(healthStatus, timeout = this.DEFAULT_TIMEOUT, allowRuntimeProbe = true) {
|
|
964
|
-
const ollamaBase = this.getOllamaBaseUrl();
|
|
1068
|
+
static async checkOllamaConfig(healthStatus, timeout = this.DEFAULT_TIMEOUT, allowRuntimeProbe = true, ollamaContext) {
|
|
1069
|
+
const ollamaBase = ollamaContext?.url ?? this.getOllamaBaseUrl();
|
|
965
1070
|
if (!ollamaBase.startsWith("http")) {
|
|
966
1071
|
healthStatus.isConfigured = false;
|
|
967
1072
|
healthStatus.configurationIssues.push("Invalid OLLAMA_BASE_URL format (OLLAMA_API_BASE is still accepted as a legacy alias)");
|
|
@@ -978,14 +1083,22 @@ export class ProviderHealthChecker {
|
|
|
978
1083
|
return { ran: false, failed: false };
|
|
979
1084
|
}
|
|
980
1085
|
const availability = await this.checkOllamaAvailability({
|
|
981
|
-
model: this.getConfiguredOllamaModel(),
|
|
1086
|
+
model: ollamaContext?.model ?? this.getConfiguredOllamaModel(),
|
|
982
1087
|
timeout,
|
|
1088
|
+
context: ollamaContext,
|
|
1089
|
+
recordDiagnostic: false,
|
|
983
1090
|
});
|
|
984
1091
|
if (!availability.available) {
|
|
985
1092
|
healthStatus.isConfigured = false;
|
|
986
1093
|
healthStatus.configurationIssues.push(`Ollama runtime check failed: ${availability.reason ?? "unknown error"}`);
|
|
987
1094
|
healthStatus.recommendations.push("Start Ollama and install the configured model before using Ollama as a fallback provider");
|
|
988
|
-
return {
|
|
1095
|
+
return {
|
|
1096
|
+
ran: availability.diagnosticOutcome === "success" ||
|
|
1097
|
+
availability.diagnosticOutcome === "transport-failure" ||
|
|
1098
|
+
availability.diagnosticOutcome === "protocol-failure",
|
|
1099
|
+
failed: availability.diagnosticOutcome === "transport-failure" ||
|
|
1100
|
+
availability.diagnosticOutcome === "protocol-failure",
|
|
1101
|
+
};
|
|
989
1102
|
}
|
|
990
1103
|
healthStatus.isConfigured = true;
|
|
991
1104
|
return { ran: true, failed: false };
|
|
@@ -1055,16 +1168,29 @@ export class ProviderHealthChecker {
|
|
|
1055
1168
|
}
|
|
1056
1169
|
}
|
|
1057
1170
|
/**
|
|
1058
|
-
*
|
|
1171
|
+
* Return health within the caller's TTL and, for Ollama, the captured identity.
|
|
1172
|
+
* Ollama revision/generation changes invalidate cached status; reading the
|
|
1173
|
+
* snapshot may apply timed expiry but never records a probe outcome.
|
|
1059
1174
|
*/
|
|
1060
|
-
static getCachedHealth(providerName, maxAge) {
|
|
1061
|
-
const
|
|
1175
|
+
static getCachedHealth(providerName, maxAge, ollamaContext) {
|
|
1176
|
+
const key = ollamaContext
|
|
1177
|
+
? getOllamaHealthCacheKey(ollamaContext)
|
|
1178
|
+
: providerName;
|
|
1179
|
+
const cached = this.healthCache.get(key);
|
|
1062
1180
|
if (!cached) {
|
|
1063
1181
|
return null;
|
|
1064
1182
|
}
|
|
1183
|
+
if (ollamaContext) {
|
|
1184
|
+
const snapshot = getOllamaDiagnosticSnapshot(ollamaContext);
|
|
1185
|
+
if (cached.ollamaRevision !== snapshot.revision ||
|
|
1186
|
+
cached.ollamaGeneration !== snapshot.generation) {
|
|
1187
|
+
this.healthCache.delete(key);
|
|
1188
|
+
return null;
|
|
1189
|
+
}
|
|
1190
|
+
}
|
|
1065
1191
|
const age = Date.now() - cached.timestamp;
|
|
1066
1192
|
if (age > maxAge) {
|
|
1067
|
-
this.healthCache.delete(
|
|
1193
|
+
this.healthCache.delete(key);
|
|
1068
1194
|
return null;
|
|
1069
1195
|
}
|
|
1070
1196
|
return cached.status;
|
|
@@ -1472,16 +1598,29 @@ export class ProviderHealthChecker {
|
|
|
1472
1598
|
backgroundHealthCheck();
|
|
1473
1599
|
}
|
|
1474
1600
|
/**
|
|
1475
|
-
* Clear health
|
|
1601
|
+
* Clear cached health and failure state for one provider or every provider.
|
|
1602
|
+
* Clearing Ollama removes all endpoint/model cache entries and explicitly
|
|
1603
|
+
* advances diagnostic generation so older in-flight permits cannot commit.
|
|
1476
1604
|
*/
|
|
1477
1605
|
static clearHealthCache(providerName) {
|
|
1478
1606
|
if (providerName) {
|
|
1479
|
-
|
|
1607
|
+
if (providerName === AIProviderName.OLLAMA) {
|
|
1608
|
+
for (const [key, cached] of this.healthCache) {
|
|
1609
|
+
if (cached.status.provider === providerName) {
|
|
1610
|
+
this.healthCache.delete(key);
|
|
1611
|
+
}
|
|
1612
|
+
}
|
|
1613
|
+
clearOllamaDiagnostics();
|
|
1614
|
+
}
|
|
1615
|
+
else {
|
|
1616
|
+
this.healthCache.delete(providerName);
|
|
1617
|
+
}
|
|
1480
1618
|
this.consecutiveFailures.delete(providerName);
|
|
1481
1619
|
}
|
|
1482
1620
|
else {
|
|
1483
1621
|
this.healthCache.clear();
|
|
1484
1622
|
this.consecutiveFailures.clear();
|
|
1623
|
+
clearOllamaDiagnostics();
|
|
1485
1624
|
}
|
|
1486
1625
|
}
|
|
1487
1626
|
static async checkFallbackProviderAvailability(providerName, model) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@juspay/neurolink",
|
|
3
|
-
"version": "12.56.
|
|
3
|
+
"version": "12.56.1",
|
|
4
4
|
"packageManager": "pnpm@10.15.1",
|
|
5
5
|
"description": "The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002/call) for routing, tool selection and context budgeting. MCP-native (4 transports), voice TTS/STT/realtime, RAG, agents, memory, compaction, 9 observability exporters. OpenAI · Anthropic · Gemini · Bedrock · Azure · Ollama · TypeSafe Jev and more.",
|
|
6
6
|
"author": {
|
|
@@ -156,6 +156,7 @@
|
|
|
156
156
|
"test:catalog-auth-header-style": "pnpm exec tsx test/continuous-test-suite-catalog-auth-header-style.ts",
|
|
157
157
|
"test:decide-catalog-schema": "pnpm exec tsx test/continuous-test-suite-decide-catalog-schema.ts",
|
|
158
158
|
"test:provider-descriptors": "pnpm exec tsx test/continuous-test-suite-provider-descriptors.ts",
|
|
159
|
+
"test:ollama-diagnostics": "pnpm exec tsx test/continuous-test-suite-ollama-diagnostics.ts",
|
|
159
160
|
"test:decide-engine": "pnpm exec tsx test/continuous-test-suite-decide-engine.ts",
|
|
160
161
|
"test:provider-structure": "pnpm exec tsx test/continuous-test-suite-provider-structure.ts",
|
|
161
162
|
"test:model-manifests": "pnpm exec tsx test/continuous-test-suite-model-manifests.ts",
|
|
@@ -163,6 +164,7 @@
|
|
|
163
164
|
"test:provider-fallback": "pnpm exec tsx test/continuous-test-suite-provider-fallback.ts",
|
|
164
165
|
"test:provider-fallback-latency": "pnpm exec tsx test/continuous-test-suite-provider-fallback-latency.ts",
|
|
165
166
|
"test:provider-wiring": "pnpm exec tsx test/continuous-test-suite-provider-wiring.ts",
|
|
167
|
+
"test:orchestrated-provider-selection": "pnpm exec tsx test/continuous-test-suite-orchestrated-provider-selection.ts",
|
|
166
168
|
"test:embedding-response-validation": "pnpm exec tsx test/continuous-test-suite-embedding-response-validation.ts",
|
|
167
169
|
"test:xiaomi-mimo-pilot": "pnpm exec tsx test/continuous-test-suite-xiaomi-mimo-pilot.ts",
|
|
168
170
|
"test:mimo-overflow-budget": "pnpm exec tsx test/continuous-test-suite-mimo-overflow-budget.ts",
|