nexus-ai-pro 2.1.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/API_STABILITY.md +38 -0
- package/CHANGELOG.md +145 -1
- package/README.md +13 -8
- package/dist/cache/memory-cache.d.ts +8 -0
- package/dist/cache/memory-cache.js +9 -0
- package/dist/cache/semantic-cache.d.ts +6 -3
- package/dist/cache/semantic-cache.js +5 -2
- package/dist/cli/args.js +3 -0
- package/dist/cli/db.d.ts +16 -4
- package/dist/cli/db.js +169 -22
- package/dist/cli/doctor.d.ts +17 -0
- package/dist/cli/doctor.js +53 -0
- package/dist/cli.js +12 -1
- package/dist/core/lifecycle.js +11 -6
- package/dist/core/nexus.d.ts +6 -0
- package/dist/core/nexus.js +103 -22
- package/dist/doctor/index.d.ts +90 -0
- package/dist/doctor/index.js +237 -0
- package/dist/graph/functional.d.ts +80 -7
- package/dist/graph/functional.js +138 -35
- package/dist/graph/graph.d.ts +15 -5
- package/dist/graph/graph.js +72 -20
- package/dist/images/evals.js +4 -2
- package/dist/operations/adapters.d.ts +52 -4
- package/dist/operations/adapters.js +194 -7
- package/dist/operations/runner.js +1 -0
- package/dist/operations/stats.js +8 -1
- package/dist/operations/store.d.ts +5 -1
- package/dist/operations/store.js +7 -0
- package/dist/ops/circuit-breaker.d.ts +52 -0
- package/dist/ops/circuit-breaker.js +4 -0
- package/dist/ops/circuit-store.d.ts +13 -2
- package/dist/ops/circuit-store.js +68 -0
- package/dist/ops/circuit-sync.d.ts +17 -0
- package/dist/ops/circuit-sync.js +86 -0
- package/dist/ops/rate-limiter.d.ts +2 -0
- package/dist/ops/rate-limiter.js +2 -0
- package/dist/postgres/circuits.d.ts +21 -4
- package/dist/postgres/circuits.js +65 -4
- package/dist/postgres/client.d.ts +0 -2
- package/dist/postgres/client.js +0 -4
- package/dist/postgres/evaluate.d.ts +16 -4
- package/dist/postgres/evaluate.js +17 -5
- package/dist/postgres/index.d.ts +32 -11
- package/dist/postgres/index.js +65 -31
- package/dist/postgres/migrations.d.ts +60 -0
- package/dist/postgres/migrations.js +121 -0
- package/dist/postgres/operations.d.ts +14 -3
- package/dist/postgres/operations.js +45 -15
- package/dist/postgres/prompts.d.ts +11 -2
- package/dist/postgres/prompts.js +14 -3
- package/dist/postgres/rollups.d.ts +35 -0
- package/dist/postgres/rollups.js +129 -0
- package/dist/postgres/store.d.ts +11 -2
- package/dist/postgres/store.js +14 -3
- package/dist/postgres/traces.d.ts +13 -3
- package/dist/postgres/traces.js +25 -3
- package/dist/postgres/vectors.d.ts +11 -2
- package/dist/postgres/vectors.js +14 -3
- package/dist/router/auto-router.d.ts +34 -7
- package/dist/router/auto-router.js +201 -68
- package/dist/server/assistant.d.ts +4 -0
- package/dist/server/assistant.js +8 -0
- package/dist/server/errors.d.ts +12 -0
- package/dist/server/errors.js +7 -0
- package/dist/server/index.d.ts +1 -1
- package/dist/server/index.js +1 -1
- package/dist/server/runs.d.ts +24 -1
- package/dist/server/runs.js +90 -14
- package/dist/sqlite/index.d.ts +16 -3
- package/dist/sqlite/index.js +16 -3
- package/dist/sqlite/migrations.d.ts +26 -0
- package/dist/sqlite/migrations.js +63 -0
- package/dist/sqlite/operations.d.ts +14 -3
- package/dist/sqlite/operations.js +42 -15
- package/dist/sqlite/store.d.ts +9 -2
- package/dist/sqlite/store.js +13 -3
- package/dist/sqlite/vectors.d.ts +11 -2
- package/dist/sqlite/vectors.js +13 -2
- package/dist/store/tenant.d.ts +12 -0
- package/dist/store/tenant.js +29 -0
- package/dist/tenancy/index.d.ts +113 -0
- package/dist/tenancy/index.js +233 -0
- package/dist/tracing/instrument.js +2 -0
- package/dist/types/config.d.ts +24 -2
- package/dist/types/graph.d.ts +11 -1
- package/dist/types/messages.d.ts +5 -0
- package/dist/types/operations.d.ts +6 -0
- package/dist/types/response.d.ts +20 -0
- package/dist/types/server.d.ts +10 -1
- package/dist/utils/schema-migrations.d.ts +97 -0
- package/dist/utils/schema-migrations.js +91 -0
- package/dist/utils/signals.d.ts +17 -0
- package/dist/utils/signals.js +26 -0
- package/dist/utils/tenant.d.ts +14 -0
- package/dist/utils/tenant.js +16 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist-cjs/cache/memory-cache.d.ts +8 -0
- package/dist-cjs/cache/memory-cache.js +9 -0
- package/dist-cjs/cache/semantic-cache.d.ts +6 -3
- package/dist-cjs/cache/semantic-cache.js +5 -2
- package/dist-cjs/cli/args.js +3 -0
- package/dist-cjs/cli/db.d.ts +16 -4
- package/dist-cjs/cli/db.js +200 -20
- package/dist-cjs/cli/doctor.d.ts +17 -0
- package/dist-cjs/cli/doctor.js +56 -0
- package/dist-cjs/core/lifecycle.js +11 -6
- package/dist-cjs/core/nexus.d.ts +6 -0
- package/dist-cjs/core/nexus.js +103 -22
- package/dist-cjs/doctor/index.d.ts +90 -0
- package/dist-cjs/doctor/index.js +276 -0
- package/dist-cjs/graph/functional.d.ts +80 -7
- package/dist-cjs/graph/functional.js +137 -34
- package/dist-cjs/graph/graph.d.ts +15 -5
- package/dist-cjs/graph/graph.js +72 -20
- package/dist-cjs/images/evals.js +4 -2
- package/dist-cjs/operations/adapters.d.ts +52 -4
- package/dist-cjs/operations/adapters.js +193 -6
- package/dist-cjs/operations/runner.js +1 -0
- package/dist-cjs/operations/stats.js +8 -1
- package/dist-cjs/operations/store.d.ts +5 -1
- package/dist-cjs/operations/store.js +7 -0
- package/dist-cjs/ops/circuit-breaker.d.ts +52 -0
- package/dist-cjs/ops/circuit-breaker.js +4 -0
- package/dist-cjs/ops/circuit-store.d.ts +13 -2
- package/dist-cjs/ops/circuit-store.js +68 -0
- package/dist-cjs/ops/circuit-sync.d.ts +17 -0
- package/dist-cjs/ops/circuit-sync.js +87 -0
- package/dist-cjs/ops/rate-limiter.d.ts +2 -0
- package/dist-cjs/ops/rate-limiter.js +2 -0
- package/dist-cjs/postgres/circuits.d.ts +21 -4
- package/dist-cjs/postgres/circuits.js +98 -3
- package/dist-cjs/postgres/client.d.ts +0 -2
- package/dist-cjs/postgres/client.js +0 -5
- package/dist-cjs/postgres/evaluate.d.ts +16 -4
- package/dist-cjs/postgres/evaluate.js +50 -4
- package/dist-cjs/postgres/index.d.ts +32 -11
- package/dist-cjs/postgres/index.js +69 -20
- package/dist-cjs/postgres/migrations.d.ts +60 -0
- package/dist-cjs/postgres/migrations.js +129 -0
- package/dist-cjs/postgres/operations.d.ts +14 -3
- package/dist-cjs/postgres/operations.js +78 -14
- package/dist-cjs/postgres/prompts.d.ts +11 -2
- package/dist-cjs/postgres/prompts.js +47 -2
- package/dist-cjs/postgres/rollups.d.ts +35 -0
- package/dist-cjs/postgres/rollups.js +168 -0
- package/dist-cjs/postgres/store.d.ts +11 -2
- package/dist-cjs/postgres/store.js +47 -2
- package/dist-cjs/postgres/traces.d.ts +13 -3
- package/dist-cjs/postgres/traces.js +58 -2
- package/dist-cjs/postgres/vectors.d.ts +11 -2
- package/dist-cjs/postgres/vectors.js +47 -2
- package/dist-cjs/router/auto-router.d.ts +34 -7
- package/dist-cjs/router/auto-router.js +201 -68
- package/dist-cjs/server/assistant.d.ts +4 -0
- package/dist-cjs/server/assistant.js +8 -0
- package/dist-cjs/server/errors.d.ts +12 -0
- package/dist-cjs/server/errors.js +9 -1
- package/dist-cjs/server/index.d.ts +1 -1
- package/dist-cjs/server/index.js +2 -1
- package/dist-cjs/server/runs.d.ts +24 -1
- package/dist-cjs/server/runs.js +89 -13
- package/dist-cjs/sqlite/index.d.ts +16 -3
- package/dist-cjs/sqlite/index.js +31 -10
- package/dist-cjs/sqlite/migrations.d.ts +26 -0
- package/dist-cjs/sqlite/migrations.js +71 -0
- package/dist-cjs/sqlite/operations.d.ts +14 -3
- package/dist-cjs/sqlite/operations.js +76 -15
- package/dist-cjs/sqlite/store.d.ts +9 -2
- package/dist-cjs/sqlite/store.js +47 -3
- package/dist-cjs/sqlite/vectors.d.ts +11 -2
- package/dist-cjs/sqlite/vectors.js +47 -2
- package/dist-cjs/store/tenant.d.ts +12 -0
- package/dist-cjs/store/tenant.js +33 -0
- package/dist-cjs/tenancy/index.d.ts +113 -0
- package/dist-cjs/tenancy/index.js +250 -0
- package/dist-cjs/tracing/instrument.js +2 -0
- package/dist-cjs/types/config.d.ts +24 -2
- package/dist-cjs/types/graph.d.ts +11 -1
- package/dist-cjs/types/messages.d.ts +5 -0
- package/dist-cjs/types/operations.d.ts +6 -0
- package/dist-cjs/types/response.d.ts +20 -0
- package/dist-cjs/types/server.d.ts +10 -1
- package/dist-cjs/utils/schema-migrations.d.ts +97 -0
- package/dist-cjs/utils/schema-migrations.js +102 -0
- package/dist-cjs/utils/signals.d.ts +17 -0
- package/dist-cjs/utils/signals.js +29 -0
- package/dist-cjs/utils/tenant.d.ts +14 -0
- package/dist-cjs/utils/tenant.js +22 -0
- package/dist-cjs/version.d.ts +1 -1
- package/dist-cjs/version.js +1 -1
- package/llms.txt +4 -3
- package/package.json +71 -2
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { diagnose } from '../doctor/index.js';
|
|
2
|
+
import { flagBool, isRecord, loadModule, numberFlag, stringFlag, writeJson } from './args.js';
|
|
3
|
+
const LABELS = { ok: 'ok ', warn: 'warn', fail: 'FAIL', skip: 'skip' };
|
|
4
|
+
export async function runDoctorCommand({ flags }) {
|
|
5
|
+
const location = stringFlag(flags, 'module');
|
|
6
|
+
const loaded = location ? await loadModule(location) : {};
|
|
7
|
+
const options = {};
|
|
8
|
+
const source = isRecord(loaded.database) ? loaded.database : loaded.client ? loaded : undefined;
|
|
9
|
+
if (source?.client) {
|
|
10
|
+
options.database = {
|
|
11
|
+
client: source.client,
|
|
12
|
+
...(source.dialect === 'postgres' || source.dialect === 'sqlite' ? { dialect: source.dialect } : {}),
|
|
13
|
+
...(Array.isArray(source.migrations) ? { migrations: source.migrations } : {}),
|
|
14
|
+
...(typeof source.table === 'string' ? { table: source.table } : {}),
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
if (loaded.redis)
|
|
18
|
+
options.redis = loaded.redis;
|
|
19
|
+
if (loaded.operations)
|
|
20
|
+
options.operations = loaded.operations;
|
|
21
|
+
if (loaded.ai)
|
|
22
|
+
options.ai = loaded.ai;
|
|
23
|
+
if (isRecord(loaded.deployment))
|
|
24
|
+
options.deployment = loaded.deployment;
|
|
25
|
+
const replicas = numberFlag(flags, 'replicas');
|
|
26
|
+
if (replicas !== undefined)
|
|
27
|
+
options.deployment = { ...options.deployment, replicas };
|
|
28
|
+
const maxQueueAgeMs = numberFlag(flags, 'max-queue-age-ms');
|
|
29
|
+
if (maxQueueAgeMs !== undefined)
|
|
30
|
+
options.maxQueueAgeMs = maxQueueAgeMs;
|
|
31
|
+
let report;
|
|
32
|
+
try {
|
|
33
|
+
report = await diagnose(options);
|
|
34
|
+
}
|
|
35
|
+
finally {
|
|
36
|
+
if (typeof loaded.close === 'function')
|
|
37
|
+
await loaded.close();
|
|
38
|
+
}
|
|
39
|
+
if (flagBool(flags, 'json'))
|
|
40
|
+
writeJson(report);
|
|
41
|
+
else
|
|
42
|
+
print(report);
|
|
43
|
+
if (!report.ok || (flagBool(flags, 'strict') && report.warnings > 0))
|
|
44
|
+
process.exitCode = 1;
|
|
45
|
+
}
|
|
46
|
+
function print(report) {
|
|
47
|
+
for (const check of report.checks) {
|
|
48
|
+
console.log(`${LABELS[check.status]} ${check.title}: ${check.detail}`);
|
|
49
|
+
if (check.hint && (check.status === 'fail' || check.status === 'warn'))
|
|
50
|
+
console.log(` ${check.hint}`);
|
|
51
|
+
}
|
|
52
|
+
console.log(`\n${report.failures} failed, ${report.warnings} warned, ${report.checks.length - report.failures - report.warnings} passed or skipped.`);
|
|
53
|
+
}
|
package/dist/cli.js
CHANGED
|
@@ -43,6 +43,11 @@ async function main() {
|
|
|
43
43
|
await runTracesCommand(subcommand, { positionals: args, flags: parsed.flags });
|
|
44
44
|
return;
|
|
45
45
|
}
|
|
46
|
+
case 'doctor': {
|
|
47
|
+
const { runDoctorCommand } = await import('./cli/doctor.js');
|
|
48
|
+
await runDoctorCommand({ positionals: parsed.positionals.slice(1), flags: parsed.flags });
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
46
51
|
case 'db': {
|
|
47
52
|
const [subcommand, ...args] = parsed.positionals;
|
|
48
53
|
const { runDbCommand } = await import('./cli/db.js');
|
|
@@ -406,8 +411,11 @@ Usage:
|
|
|
406
411
|
[--min-cost n] [--feedback-key k] [--limit 20] [--offset n] [--json]
|
|
407
412
|
nexus traces show <traceId> --store <runs.jsonl|store.mjs> [--json]
|
|
408
413
|
nexus traces export --store <runs.jsonl|store.mjs> [--out runs.jsonl] [same filters as list]
|
|
409
|
-
nexus db sql [--adapters operations,store,traces,evaluation,circuits,prompts] [--vector-dimensions 1536]
|
|
414
|
+
nexus db sql [--adapters operations,store,traces,evaluation,circuits,prompts,rollups] [--vector-dimensions 1536] [--record]
|
|
415
|
+
nexus db status --client <db.mjs> [--adapters ...] [--dialect postgres|sqlite] [--table t] [--check] [--json]
|
|
416
|
+
nexus db migrate --client <db.mjs> [--dry-run] [--adapters ...] [--dialect postgres|sqlite] [--table t] [--json]
|
|
410
417
|
nexus migrate [paths...] [--write] [--check] [--json]
|
|
418
|
+
nexus doctor [--module doctor.mjs] [--replicas n] [--max-queue-age-ms n] [--strict] [--json]
|
|
411
419
|
nexus graph lint <module>[#export] [--deployed] [--strict] [--ignore CODE,CODE] [--json]
|
|
412
420
|
nexus deploy status [assistant] --url <server> [--token t] [--json]
|
|
413
421
|
nexus deploy canary <assistant> <revision> <percent> --url <server> [--reason r] [--version n]
|
|
@@ -419,6 +427,9 @@ Eval files can export { cases, client } or { cases, config }. JSON cases may use
|
|
|
419
427
|
Evaluation modules for "eval run" export { target, dataset, evaluators, summary?, options? }.
|
|
420
428
|
"eval gate" exits 1 when a metric got worse beyond noise, a new failure appeared, or the datasets differ.
|
|
421
429
|
Trace store modules export any TraceStore as { store }, such as a PostgresTraceStore over the application's own pool.
|
|
430
|
+
Database modules for "db status" and "db migrate" export { client }, and optionally { dialect, transaction, migrations, close }.
|
|
431
|
+
"db status --check" exits 1 while a migration is pending or changed.
|
|
432
|
+
"doctor" exits 1 when a check fails, or with --strict when one warns. Its module exports any of { database, client, redis, operations, ai, deployment, close }.
|
|
422
433
|
"deploy" reads NEXUS_SERVER_URL and NEXUS_SERVER_TOKEN when --url and --token are not given.
|
|
423
434
|
"migrate" moves 1.x root imports to their 2.0 subpaths and reports removed options; --check exits 1 while any remain.
|
|
424
435
|
"graph lint" reads a compiled graph's shape and exits 1 on an error finding, or on a warning with --strict.`);
|
package/dist/core/lifecycle.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { NexusRateLimitError } from '../ops/rate-limiter.js';
|
|
2
2
|
import { generateRequestId } from '../utils/ids.js';
|
|
3
|
+
import { linkSignals } from '../utils/signals.js';
|
|
3
4
|
export const LIFECYCLE_STAGES = [
|
|
4
5
|
'validate',
|
|
5
6
|
'authorize',
|
|
@@ -43,14 +44,15 @@ export class OperationLifecycle {
|
|
|
43
44
|
const startedAt = Date.now();
|
|
44
45
|
const timeoutMs = options.timeoutMs ?? this.runtime.timeoutMs;
|
|
45
46
|
const deadline = timeoutMs !== undefined && Number.isFinite(timeoutMs) && timeoutMs > 0 ? startedAt + timeoutMs : undefined;
|
|
47
|
+
const call = callSignal(options.signal, deadline === undefined ? undefined : deadline - startedAt);
|
|
46
48
|
const context = {
|
|
47
49
|
requestId: descriptor.requestId,
|
|
48
|
-
signal:
|
|
50
|
+
signal: call.signal,
|
|
49
51
|
...(deadline !== undefined ? { deadline } : {}),
|
|
50
52
|
...(options.traceContext ? { traceContext: options.traceContext } : {}),
|
|
51
53
|
...(options.idempotencyKey ? { idempotencyKey: options.idempotencyKey } : {}),
|
|
52
54
|
};
|
|
53
|
-
const ticket = new Ticket(this.runtime, descriptor, context, startedAt);
|
|
55
|
+
const ticket = new Ticket(this.runtime, descriptor, context, startedAt, call.dispose);
|
|
54
56
|
try {
|
|
55
57
|
await this.runtime.metrics?.recordRequest(labelsOf(descriptor));
|
|
56
58
|
await this.checkRateLimit(descriptor);
|
|
@@ -105,6 +107,7 @@ export class OperationLifecycle {
|
|
|
105
107
|
const bucket = {
|
|
106
108
|
model: descriptor.model ?? `${descriptor.family}:${descriptor.operation}`,
|
|
107
109
|
userId: descriptor.userId,
|
|
110
|
+
tenantId: descriptor.tenantId,
|
|
108
111
|
};
|
|
109
112
|
if (rateLimit.store)
|
|
110
113
|
await rateLimiter.checkAsync(bucket, rateLimit);
|
|
@@ -135,13 +138,15 @@ class Ticket {
|
|
|
135
138
|
descriptor;
|
|
136
139
|
context;
|
|
137
140
|
startedAt;
|
|
141
|
+
release;
|
|
138
142
|
reservation;
|
|
139
143
|
finished = false;
|
|
140
|
-
constructor(runtime, descriptor, context, startedAt) {
|
|
144
|
+
constructor(runtime, descriptor, context, startedAt, release = () => undefined) {
|
|
141
145
|
this.runtime = runtime;
|
|
142
146
|
this.descriptor = descriptor;
|
|
143
147
|
this.context = context;
|
|
144
148
|
this.startedAt = startedAt;
|
|
149
|
+
this.release = release;
|
|
145
150
|
}
|
|
146
151
|
async reserve(estimate) {
|
|
147
152
|
const budget = this.runtime.config?.budget;
|
|
@@ -211,14 +216,14 @@ class Ticket {
|
|
|
211
216
|
});
|
|
212
217
|
}
|
|
213
218
|
async finish(outcome) {
|
|
219
|
+
this.release();
|
|
214
220
|
await runHook(() => this.runtime.config?.hooks?.onFinish?.(this.descriptor, outcome));
|
|
215
221
|
}
|
|
216
222
|
}
|
|
217
223
|
function callSignal(signal, timeoutMs) {
|
|
218
224
|
if (timeoutMs === undefined)
|
|
219
|
-
return signal ?? new AbortController().signal;
|
|
220
|
-
|
|
221
|
-
return signal ? AbortSignal.any([signal, timeout]) : timeout;
|
|
225
|
+
return { signal: signal ?? new AbortController().signal, dispose: () => undefined };
|
|
226
|
+
return linkSignals([signal, AbortSignal.timeout(timeoutMs)]);
|
|
222
227
|
}
|
|
223
228
|
function outcomeOf(error, signal) {
|
|
224
229
|
if (error instanceof OperationDeniedError ||
|
package/dist/core/nexus.d.ts
CHANGED
|
@@ -318,6 +318,12 @@ export declare class NexusAI {
|
|
|
318
318
|
private attachTrace;
|
|
319
319
|
private usesSemanticCache;
|
|
320
320
|
private loadSemanticCache;
|
|
321
|
+
/**
|
|
322
|
+
* Where a request's cached response lives: its key, its namespace, and the scope a semantic match
|
|
323
|
+
* must share, which is always at least the request's tenant.
|
|
324
|
+
*/
|
|
325
|
+
private cachePlan;
|
|
326
|
+
/** Looks a request up in the cache. A cache that fails is reported as an `error` outcome and treated as a miss. */
|
|
321
327
|
private getCachedResponse;
|
|
322
328
|
private setCachedResponse;
|
|
323
329
|
private estimateRequestCost;
|
package/dist/core/nexus.js
CHANGED
|
@@ -154,13 +154,26 @@ export class NexusAI {
|
|
|
154
154
|
}
|
|
155
155
|
await this.applyBudgets(context, ticket);
|
|
156
156
|
const responseFormat = context.request.responseFormat || this.config.responseFormat;
|
|
157
|
-
const
|
|
158
|
-
|
|
159
|
-
|
|
157
|
+
const cachePlan = this.isCacheEnabled() ? this.cachePlan(context.request, responseFormat) : undefined;
|
|
158
|
+
let cacheOutcome;
|
|
159
|
+
if (cachePlan && !cachePlan.read) {
|
|
160
|
+
cacheOutcome = {
|
|
161
|
+
outcome: 'bypass',
|
|
162
|
+
reason: context.request.responseCache === 'refresh'
|
|
163
|
+
? 'the request asked for a fresh answer'
|
|
164
|
+
: 'the request skipped the cache',
|
|
165
|
+
...(cachePlan.namespace ? { namespace: cachePlan.namespace } : {}),
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
if (cachePlan?.read) {
|
|
169
|
+
const plan = cachePlan;
|
|
170
|
+
const lookup = await this.pipeline.trace(context, 'cacheLookup', () => this.getCachedResponse(plan, context.request));
|
|
171
|
+
cacheOutcome = lookup.outcome;
|
|
172
|
+
const cached = lookup.response;
|
|
160
173
|
if (cached) {
|
|
161
174
|
let cachedResponse = this.attachTrace({
|
|
162
175
|
...cached,
|
|
163
|
-
meta: { ...cached.meta, requestId: ticket.descriptor.requestId, cacheHit: true },
|
|
176
|
+
meta: { ...cached.meta, requestId: ticket.descriptor.requestId, cacheHit: true, cache: lookup.outcome },
|
|
164
177
|
}, this.pipeline.finish(context));
|
|
165
178
|
if (this.isSecurityEnabled()) {
|
|
166
179
|
const outputResult = this.security.protectOutput(cachedResponse);
|
|
@@ -222,12 +235,24 @@ export class NexusAI {
|
|
|
222
235
|
});
|
|
223
236
|
context.response = formattedResponse;
|
|
224
237
|
}
|
|
225
|
-
if (
|
|
238
|
+
if (cachePlan?.write) {
|
|
226
239
|
const responseToCache = context.response;
|
|
227
240
|
if (!responseToCache) {
|
|
228
241
|
throw new Error('Cannot cache an empty response.');
|
|
229
242
|
}
|
|
230
|
-
|
|
243
|
+
const plan = cachePlan;
|
|
244
|
+
try {
|
|
245
|
+
await this.pipeline.trace(context, 'cacheWrite', () => this.setCachedResponse(plan, context.request, responseToCache));
|
|
246
|
+
}
|
|
247
|
+
catch (error) {
|
|
248
|
+
cacheOutcome = {
|
|
249
|
+
...(cacheOutcome ?? { outcome: 'miss', reason: 'no entry for this request' }),
|
|
250
|
+
writeError: error instanceof Error ? error.message : String(error),
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
if (cacheOutcome && context.response) {
|
|
255
|
+
context.response = { ...context.response, meta: { ...context.response.meta, cache: cacheOutcome } };
|
|
231
256
|
}
|
|
232
257
|
context = await this.pipeline.runHook('beforeReturn', context);
|
|
233
258
|
let responseToReturn = context.response;
|
|
@@ -794,35 +819,86 @@ export class NexusAI {
|
|
|
794
819
|
});
|
|
795
820
|
return this.semanticCacheLoading;
|
|
796
821
|
}
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
822
|
+
cachePlan(request, responseFormat) {
|
|
823
|
+
const configured = this.config.cache?.namespace;
|
|
824
|
+
const chosen = typeof configured === 'function' ? configured(request) : configured;
|
|
825
|
+
const namespace = chosen ? namespaceKey({ ...chosen, tenant: chosen.tenant ?? request.tenantId }) : undefined;
|
|
826
|
+
const base = createCacheKey({ request: { ...request, responseCache: undefined }, responseFormat });
|
|
827
|
+
return {
|
|
828
|
+
key: namespace ? `${namespace}\u0000${base}` : base,
|
|
829
|
+
...(namespace ? { namespace } : {}),
|
|
830
|
+
scope: namespace ?? namespaceKey({ tenant: request.tenantId }),
|
|
831
|
+
read: request.responseCache === undefined,
|
|
832
|
+
write: request.responseCache !== 'bypass',
|
|
833
|
+
};
|
|
834
|
+
}
|
|
835
|
+
async getCachedResponse(plan, request) {
|
|
836
|
+
const namespace = plan.namespace ? { namespace: plan.namespace } : {};
|
|
837
|
+
const cache = this.config.cache ?? {};
|
|
838
|
+
let layer = this.usesSemanticCache() ? 'semantic' : 'exact';
|
|
839
|
+
try {
|
|
840
|
+
if (this.usesSemanticCache()) {
|
|
841
|
+
const semantic = await (await this.loadSemanticCache()).get(request, plan.scope);
|
|
842
|
+
if (semantic) {
|
|
843
|
+
return {
|
|
844
|
+
response: semantic,
|
|
845
|
+
outcome: { outcome: 'hit', layer: 'semantic', reason: 'a similar request was answered', ...namespace },
|
|
846
|
+
};
|
|
847
|
+
}
|
|
848
|
+
if (cache.strategy === 'semantic') {
|
|
849
|
+
return {
|
|
850
|
+
outcome: {
|
|
851
|
+
outcome: 'miss',
|
|
852
|
+
layer: 'semantic',
|
|
853
|
+
reason: 'no similar request above the threshold',
|
|
854
|
+
...namespace,
|
|
855
|
+
},
|
|
856
|
+
};
|
|
857
|
+
}
|
|
858
|
+
}
|
|
859
|
+
layer = 'exact';
|
|
860
|
+
if (cache.adapter) {
|
|
861
|
+
const value = (await cache.adapter.get(plan.key));
|
|
862
|
+
return value
|
|
863
|
+
? { response: value, outcome: { outcome: 'hit', layer, reason: 'this request was answered', ...namespace } }
|
|
864
|
+
: { outcome: { outcome: 'miss', layer, reason: 'no entry for this request', ...namespace } };
|
|
865
|
+
}
|
|
866
|
+
const { value, stale } = this.cache.lookup(plan.key);
|
|
867
|
+
if (value) {
|
|
868
|
+
return {
|
|
869
|
+
response: value,
|
|
870
|
+
outcome: { outcome: 'hit', layer, reason: 'this request was answered', ...namespace },
|
|
871
|
+
};
|
|
872
|
+
}
|
|
873
|
+
return stale
|
|
874
|
+
? { outcome: { outcome: 'stale', layer, reason: 'the entry for this request had expired', ...namespace } }
|
|
875
|
+
: { outcome: { outcome: 'miss', layer, reason: 'no entry for this request', ...namespace } };
|
|
804
876
|
}
|
|
805
|
-
|
|
806
|
-
return
|
|
807
|
-
|
|
808
|
-
|
|
877
|
+
catch (error) {
|
|
878
|
+
return {
|
|
879
|
+
outcome: {
|
|
880
|
+
outcome: 'error',
|
|
881
|
+
layer,
|
|
882
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
883
|
+
...namespace,
|
|
884
|
+
},
|
|
885
|
+
};
|
|
809
886
|
}
|
|
810
|
-
return this.cache.get(cacheKey);
|
|
811
887
|
}
|
|
812
|
-
async setCachedResponse(
|
|
888
|
+
async setCachedResponse(plan, request, response) {
|
|
813
889
|
if (!this.config.cache?.enabled)
|
|
814
890
|
return;
|
|
815
891
|
const ttl = this.config.cache.ttlSeconds || 300;
|
|
816
892
|
if (this.usesSemanticCache()) {
|
|
817
|
-
await (await this.loadSemanticCache()).set(
|
|
893
|
+
await (await this.loadSemanticCache()).set(plan.key, request, response, plan.scope);
|
|
818
894
|
}
|
|
819
895
|
if (this.config.cache.strategy === 'semantic')
|
|
820
896
|
return;
|
|
821
897
|
if (this.config.cache.adapter) {
|
|
822
|
-
await this.config.cache.adapter.set(
|
|
898
|
+
await this.config.cache.adapter.set(plan.key, response, ttl);
|
|
823
899
|
return;
|
|
824
900
|
}
|
|
825
|
-
this.cache.set(
|
|
901
|
+
this.cache.set(plan.key, response, ttl);
|
|
826
902
|
}
|
|
827
903
|
estimateRequestCost(request) {
|
|
828
904
|
return estimateCost({
|
|
@@ -856,3 +932,8 @@ function streamStopped() {
|
|
|
856
932
|
error.name = 'AbortError';
|
|
857
933
|
return error;
|
|
858
934
|
}
|
|
935
|
+
function namespaceKey(namespace) {
|
|
936
|
+
return [namespace.tenant, namespace.environment, namespace.modelRevision, namespace.contextVersion]
|
|
937
|
+
.map((part) => encodeURIComponent(part ?? ''))
|
|
938
|
+
.join('/');
|
|
939
|
+
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import type { SchemaMigration } from '../utils/schema-migrations.js';
|
|
2
|
+
/** How one check came out: `fail` breaks the deployment, `warn` needs a look, `skip` had nothing to check. */
|
|
3
|
+
export type DoctorStatus = 'ok' | 'warn' | 'fail' | 'skip';
|
|
4
|
+
/** One check's result. */
|
|
5
|
+
export interface DoctorCheck {
|
|
6
|
+
/** A stable id, such as `node` or `migrations`. */
|
|
7
|
+
id: string;
|
|
8
|
+
/** What was checked, in a few words. */
|
|
9
|
+
title: string;
|
|
10
|
+
/** How it came out. */
|
|
11
|
+
status: DoctorStatus;
|
|
12
|
+
/** What was found. Never a secret's value. */
|
|
13
|
+
detail: string;
|
|
14
|
+
/** What to do about a warning or a failure. */
|
|
15
|
+
hint?: string;
|
|
16
|
+
}
|
|
17
|
+
/** Every check's result, and whether the deployment can run. */
|
|
18
|
+
export interface DoctorReport {
|
|
19
|
+
/** The checks, in the order they ran. */
|
|
20
|
+
checks: DoctorCheck[];
|
|
21
|
+
/** True when nothing failed. */
|
|
22
|
+
ok: boolean;
|
|
23
|
+
/** Checks that failed. */
|
|
24
|
+
failures: number;
|
|
25
|
+
/** Checks that warned. */
|
|
26
|
+
warnings: number;
|
|
27
|
+
}
|
|
28
|
+
/** A database for the doctor to reach, as `nexus db` reaches one. */
|
|
29
|
+
export interface DoctorDatabase {
|
|
30
|
+
/** A Postgres client (`query`) or a SQLite handle or client. */
|
|
31
|
+
client: unknown;
|
|
32
|
+
/** Which kind it is. Detected from the client when omitted. */
|
|
33
|
+
dialect?: 'postgres' | 'sqlite';
|
|
34
|
+
/** The migrations it should have. Defaults to every bundled adapter's on its default tables. */
|
|
35
|
+
migrations?: readonly SchemaMigration[];
|
|
36
|
+
/** The table that records applied migrations. */
|
|
37
|
+
table?: string;
|
|
38
|
+
}
|
|
39
|
+
/** What the doctor knows about how the application is deployed. */
|
|
40
|
+
export interface DoctorDeployment {
|
|
41
|
+
/** How many replicas run it. Settings that keep state in one process are checked only above 1. */
|
|
42
|
+
replicas: number;
|
|
43
|
+
/**
|
|
44
|
+
* The stores and shared state it uses, by any name: a checkpointer, the operation store, the
|
|
45
|
+
* server's state, tenant usage, a rate-limit store, a circuit store, a cache. Each is reported when
|
|
46
|
+
* it keeps its state in process memory, where each replica would see only its own.
|
|
47
|
+
*/
|
|
48
|
+
stores?: Record<string, unknown>;
|
|
49
|
+
}
|
|
50
|
+
/** What `diagnose()` checks. Everything is optional; a check without its input is skipped. */
|
|
51
|
+
export interface DoctorOptions {
|
|
52
|
+
/** The Node.js version to check. Defaults to the running one. */
|
|
53
|
+
nodeVersion?: string;
|
|
54
|
+
/** Where credentials are read from. Defaults to `process.env`. Only names are reported. */
|
|
55
|
+
env?: Record<string, string | undefined>;
|
|
56
|
+
/** Whether a package is installed. Defaults to resolving it from the working directory. */
|
|
57
|
+
resolvePackage?: (name: string) => boolean;
|
|
58
|
+
/** A database: connectivity, and migrations pending or changed. */
|
|
59
|
+
database?: DoctorDatabase;
|
|
60
|
+
/** A Redis client to ping. */
|
|
61
|
+
redis?: {
|
|
62
|
+
ping(): unknown;
|
|
63
|
+
};
|
|
64
|
+
/** An operation store, read for leases that lapsed and a queue nobody drains. */
|
|
65
|
+
operations?: {
|
|
66
|
+
stats?(now: string): unknown;
|
|
67
|
+
list?(): unknown;
|
|
68
|
+
};
|
|
69
|
+
/** How long the oldest queued operation may wait before the queue counts as stalled. Defaults to 15 minutes. */
|
|
70
|
+
maxQueueAgeMs?: number;
|
|
71
|
+
/** A client, read for provider health. */
|
|
72
|
+
ai?: {
|
|
73
|
+
getProviderHealth(): ReadonlyArray<{
|
|
74
|
+
providerName: string;
|
|
75
|
+
healthy: boolean;
|
|
76
|
+
stale?: boolean;
|
|
77
|
+
}>;
|
|
78
|
+
};
|
|
79
|
+
/** How old the bundled model registry may be, in days. Defaults to the registry's own window. */
|
|
80
|
+
registryMaxAgeDays?: number;
|
|
81
|
+
/** How the application is deployed. */
|
|
82
|
+
deployment?: DoctorDeployment;
|
|
83
|
+
/** Replaces the clock, for tests. */
|
|
84
|
+
now?: () => Date;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Runs every check its options allow and reports each, never throwing: a check that cannot run is a
|
|
88
|
+
* failure with the reason.
|
|
89
|
+
*/
|
|
90
|
+
export declare function diagnose(options?: DoctorOptions): Promise<DoctorReport>;
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
import { createRequire } from 'node:module';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
const PEERS = {
|
|
4
|
+
openai: 'the OpenAI SDK, used only where an application passes one in',
|
|
5
|
+
'@anthropic-ai/sdk': 'the Anthropic SDK, likewise',
|
|
6
|
+
ollama: 'the Ollama client',
|
|
7
|
+
zod: 'zod schemas for tools and structured output',
|
|
8
|
+
ajv: 'JSON Schema validation of structured output',
|
|
9
|
+
'ajv-formats': 'formats for ajv',
|
|
10
|
+
};
|
|
11
|
+
const CREDENTIALS = {
|
|
12
|
+
openai: ['OPENAI_API_KEY'],
|
|
13
|
+
anthropic: ['ANTHROPIC_API_KEY'],
|
|
14
|
+
google: ['GOOGLE_API_KEY', 'GEMINI_API_KEY'],
|
|
15
|
+
'azure-openai': ['AZURE_OPENAI_API_KEY'],
|
|
16
|
+
groq: ['GROQ_API_KEY'],
|
|
17
|
+
mistral: ['MISTRAL_API_KEY'],
|
|
18
|
+
cohere: ['COHERE_API_KEY', 'CO_API_KEY'],
|
|
19
|
+
deepseek: ['DEEPSEEK_API_KEY'],
|
|
20
|
+
openrouter: ['OPENROUTER_API_KEY'],
|
|
21
|
+
};
|
|
22
|
+
const PROCESS_LOCAL = {
|
|
23
|
+
MemoryGraphCheckpointer: { status: 'fail', why: 'a thread started on one replica cannot be resumed on another' },
|
|
24
|
+
MemoryOperationStore: { status: 'fail', why: 'operations and server runs are invisible to other replicas' },
|
|
25
|
+
MemoryServerStore: { status: 'fail', why: 'threads and runs exist only on the replica that created them' },
|
|
26
|
+
MemoryRunEventLog: { status: 'fail', why: 'a client reconnecting to another replica cannot follow its run' },
|
|
27
|
+
MemoryTenantUsage: { status: 'fail', why: 'each replica counts its own tenant limits, multiplying them' },
|
|
28
|
+
MemoryRateLimitStore: { status: 'warn', why: 'each replica enforces its own limit, multiplying it' },
|
|
29
|
+
MemoryCircuitStateStore: { status: 'warn', why: 'a circuit opened on one replica stays closed on the others' },
|
|
30
|
+
MemoryStore: { status: 'warn', why: 'long-term memory written on one replica is missing on the others' },
|
|
31
|
+
MemoryTraceStore: { status: 'warn', why: 'each replica keeps only its own traces' },
|
|
32
|
+
MemoryPromptStore: { status: 'warn', why: 'a prompt promoted on one replica is not served by the others' },
|
|
33
|
+
MemoryCacheAdapter: { status: 'warn', why: 'each replica caches separately' },
|
|
34
|
+
MemoryRollupStore: { status: 'warn', why: 'each replica sums only its own runs' },
|
|
35
|
+
};
|
|
36
|
+
export async function diagnose(options = {}) {
|
|
37
|
+
const now = options.now ?? (() => new Date());
|
|
38
|
+
const checks = [];
|
|
39
|
+
const run = async (id, title, check) => {
|
|
40
|
+
try {
|
|
41
|
+
checks.push({ id, title, ...(await check()) });
|
|
42
|
+
}
|
|
43
|
+
catch (error) {
|
|
44
|
+
checks.push({ id, title, status: 'fail', detail: error instanceof Error ? error.message : String(error) });
|
|
45
|
+
}
|
|
46
|
+
};
|
|
47
|
+
await run('node', 'Node.js version', () => nodeCheck(options.nodeVersion ?? process.versions.node));
|
|
48
|
+
await run('peers', 'Optional packages', () => peersCheck(options.resolvePackage ?? defaultResolver()));
|
|
49
|
+
await run('credentials', 'Provider credentials', () => credentialsCheck(options.env ?? process.env));
|
|
50
|
+
await run('registry', 'Model registry age', () => registryCheck(now(), options.registryMaxAgeDays));
|
|
51
|
+
await run('database', 'Database connectivity', () => databaseCheck(options.database));
|
|
52
|
+
await run('migrations', 'Schema migrations', () => migrationsCheck(options.database));
|
|
53
|
+
await run('redis', 'Redis', () => redisCheck(options.redis));
|
|
54
|
+
await run('queue', 'Operation queue and leases', () => queueCheck(options.operations, now(), options.maxQueueAgeMs));
|
|
55
|
+
await run('health', 'Provider health', () => healthCheck(options.ai));
|
|
56
|
+
await run('topology', 'Settings that only work in one process', () => topologyCheck(options.deployment));
|
|
57
|
+
const failures = checks.filter((check) => check.status === 'fail').length;
|
|
58
|
+
const warnings = checks.filter((check) => check.status === 'warn').length;
|
|
59
|
+
return { checks, ok: failures === 0, failures, warnings };
|
|
60
|
+
}
|
|
61
|
+
function nodeCheck(version) {
|
|
62
|
+
const major = Number(version.replace(/^v/, '').split('.')[0]);
|
|
63
|
+
return major >= 22
|
|
64
|
+
? { status: 'ok', detail: `Node.js ${version}` }
|
|
65
|
+
: { status: 'fail', detail: `Node.js ${version}; this package needs 22 or newer`, hint: 'Upgrade Node.js.' };
|
|
66
|
+
}
|
|
67
|
+
function defaultResolver() {
|
|
68
|
+
const resolve = createRequire(path.join(process.cwd(), 'package.json')).resolve;
|
|
69
|
+
return (name) => {
|
|
70
|
+
try {
|
|
71
|
+
resolve(name);
|
|
72
|
+
return true;
|
|
73
|
+
}
|
|
74
|
+
catch {
|
|
75
|
+
return false;
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
function peersCheck(resolve) {
|
|
80
|
+
const installed = Object.keys(PEERS).filter((name) => resolve(name));
|
|
81
|
+
const missing = Object.keys(PEERS).filter((name) => !installed.includes(name));
|
|
82
|
+
return {
|
|
83
|
+
status: 'ok',
|
|
84
|
+
detail: `Installed: ${installed.length ? installed.join(', ') : 'none'}. Not installed: ${missing.length ? missing.join(', ') : 'none'}. Every one is optional; install only what you use.`,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
function credentialsCheck(env) {
|
|
88
|
+
const found = Object.entries(CREDENTIALS)
|
|
89
|
+
.filter(([, names]) => names.some((name) => Boolean(env[name]?.trim())))
|
|
90
|
+
.map(([provider]) => provider);
|
|
91
|
+
return found.length
|
|
92
|
+
? { status: 'ok', detail: `Credentials found for ${found.join(', ')}.` }
|
|
93
|
+
: {
|
|
94
|
+
status: 'warn',
|
|
95
|
+
detail: 'No provider credential found in the environment.',
|
|
96
|
+
hint: 'Set the API key of each hosted provider you use, such as OPENAI_API_KEY. Local providers need none.',
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
async function registryCheck(now, maxAgeDays) {
|
|
100
|
+
const { checkRegistryFreshness } = await import('../models/registry.js');
|
|
101
|
+
const freshness = checkRegistryFreshness({ now, ...(maxAgeDays === undefined ? {} : { maxAgeDays }) });
|
|
102
|
+
return freshness.stale
|
|
103
|
+
? {
|
|
104
|
+
status: 'warn',
|
|
105
|
+
detail: `The bundled registry was verified ${freshness.ageDays} days ago (${freshness.verifiedAt}), past its ${freshness.maxAgeDays}-day window.`,
|
|
106
|
+
hint: 'Upgrade nexus-ai-pro, or override stale entries with models.registry.',
|
|
107
|
+
}
|
|
108
|
+
: { status: 'ok', detail: `Verified ${freshness.ageDays} days ago (${freshness.verifiedAt}).` };
|
|
109
|
+
}
|
|
110
|
+
function dialectOf(database) {
|
|
111
|
+
if (database.dialect)
|
|
112
|
+
return database.dialect;
|
|
113
|
+
const client = database.client;
|
|
114
|
+
if (client && typeof client.query === 'function')
|
|
115
|
+
return 'postgres';
|
|
116
|
+
return 'sqlite';
|
|
117
|
+
}
|
|
118
|
+
async function databaseCheck(database) {
|
|
119
|
+
if (!database)
|
|
120
|
+
return { status: 'skip', detail: 'No database given.' };
|
|
121
|
+
const dialect = dialectOf(database);
|
|
122
|
+
if (dialect === 'postgres') {
|
|
123
|
+
await database.client.query('SELECT 1');
|
|
124
|
+
}
|
|
125
|
+
else {
|
|
126
|
+
const { toSqliteClient } = await import('../sqlite/client.js');
|
|
127
|
+
await toSqliteClient(database.client).all('SELECT 1', []);
|
|
128
|
+
}
|
|
129
|
+
return { status: 'ok', detail: `Reached the ${dialect === 'postgres' ? 'Postgres' : 'SQLite'} database.` };
|
|
130
|
+
}
|
|
131
|
+
async function migrationsCheck(database) {
|
|
132
|
+
if (!database)
|
|
133
|
+
return { status: 'skip', detail: 'No database given.' };
|
|
134
|
+
const dialect = dialectOf(database);
|
|
135
|
+
const options = database.table ? { table: database.table } : {};
|
|
136
|
+
let status;
|
|
137
|
+
if (dialect === 'postgres') {
|
|
138
|
+
const [{ postgresMigrations }, { postgresMigrationStatus }] = await Promise.all([
|
|
139
|
+
import('../postgres/index.js'),
|
|
140
|
+
import('../postgres/migrations.js'),
|
|
141
|
+
]);
|
|
142
|
+
status = await postgresMigrationStatus(database.client, database.migrations ?? postgresMigrations(), options);
|
|
143
|
+
}
|
|
144
|
+
else {
|
|
145
|
+
const [{ sqliteMigrations }, { sqliteMigrationStatus }] = await Promise.all([
|
|
146
|
+
import('../sqlite/index.js'),
|
|
147
|
+
import('../sqlite/migrations.js'),
|
|
148
|
+
]);
|
|
149
|
+
status = await sqliteMigrationStatus(database.client, database.migrations ?? sqliteMigrations(), options);
|
|
150
|
+
}
|
|
151
|
+
if (status.changed.length) {
|
|
152
|
+
return {
|
|
153
|
+
status: 'fail',
|
|
154
|
+
detail: `${status.changed.length} migration${status.changed.length === 1 ? '' : 's'} changed after running: ${status.changed.map(({ migration }) => `${migration.component} v${migration.version}`).join(', ')}.`,
|
|
155
|
+
hint: 'A migration never changes once applied. Check the adapter options match those it ran with.',
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
if (status.pending.length) {
|
|
159
|
+
return {
|
|
160
|
+
status: 'fail',
|
|
161
|
+
detail: `${status.pending.length} pending: ${status.pending.map((migration) => `${migration.component} v${migration.version}`).join(', ')}.`,
|
|
162
|
+
hint: 'Run: nexus db migrate --client db.mjs',
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
const newer = status.unknown.length ? ` ${status.unknown.length} recorded by a newer release.` : '';
|
|
166
|
+
return { status: 'ok', detail: `${status.applied.length} applied, none pending.${newer}` };
|
|
167
|
+
}
|
|
168
|
+
async function redisCheck(redis) {
|
|
169
|
+
if (!redis)
|
|
170
|
+
return { status: 'skip', detail: 'No Redis client given.' };
|
|
171
|
+
const reply = await redis.ping();
|
|
172
|
+
return { status: 'ok', detail: `PING answered ${String(reply)}.` };
|
|
173
|
+
}
|
|
174
|
+
async function queueCheck(operations, now, maxQueueAgeMs = 15 * 60_000) {
|
|
175
|
+
if (!operations)
|
|
176
|
+
return { status: 'skip', detail: 'No operation store given.' };
|
|
177
|
+
const { operationStats } = await import('../operations/stats.js');
|
|
178
|
+
const stats = await operationStats(operations, { now });
|
|
179
|
+
const problems = [];
|
|
180
|
+
if (stats.lapsedLeases > 0) {
|
|
181
|
+
problems.push(`${stats.lapsedLeases} running operation${stats.lapsedLeases === 1 ? '' : 's'} hold a lease that lapsed`);
|
|
182
|
+
}
|
|
183
|
+
const age = stats.oldestQueuedAt ? now.getTime() - Date.parse(stats.oldestQueuedAt) : 0;
|
|
184
|
+
if (age > maxQueueAgeMs)
|
|
185
|
+
problems.push(`the oldest queued operation has waited ${Math.round(age / 60_000)} minutes`);
|
|
186
|
+
const counts = Object.entries(stats.byStatus)
|
|
187
|
+
.map(([status, count]) => `${count} ${status}`)
|
|
188
|
+
.join(', ');
|
|
189
|
+
return problems.length
|
|
190
|
+
? {
|
|
191
|
+
status: 'warn',
|
|
192
|
+
detail: `${problems.join('; ')}. (${counts || 'nothing unfinished'})`,
|
|
193
|
+
hint: 'Run recovery — OperationRunner.recover(), or the agent server with recoverEveryMs — and a worker that claims the queue.',
|
|
194
|
+
}
|
|
195
|
+
: { status: 'ok', detail: counts ? `Unfinished: ${counts}. No lapsed lease.` : 'Nothing unfinished.' };
|
|
196
|
+
}
|
|
197
|
+
function healthCheck(ai) {
|
|
198
|
+
if (!ai)
|
|
199
|
+
return { status: 'skip', detail: 'No client given.' };
|
|
200
|
+
const snapshots = ai.getProviderHealth();
|
|
201
|
+
if (snapshots.length === 0)
|
|
202
|
+
return { status: 'skip', detail: 'Health tracking is off, or no provider has been called.' };
|
|
203
|
+
const stale = snapshots.filter((item) => item.stale).map((item) => item.providerName);
|
|
204
|
+
const unhealthy = snapshots.filter((item) => !item.healthy && !item.stale).map((item) => item.providerName);
|
|
205
|
+
const issues = [
|
|
206
|
+
...(stale.length ? [`stale: ${stale.join(', ')}`] : []),
|
|
207
|
+
...(unhealthy.length ? [`unhealthy: ${unhealthy.join(', ')}`] : []),
|
|
208
|
+
];
|
|
209
|
+
return issues.length
|
|
210
|
+
? {
|
|
211
|
+
status: 'warn',
|
|
212
|
+
detail: `${issues.join('; ')}.`,
|
|
213
|
+
hint: 'Refresh stale entries with ai.checkProviders({ staleOnly: true }).',
|
|
214
|
+
}
|
|
215
|
+
: { status: 'ok', detail: `${snapshots.length} provider${snapshots.length === 1 ? '' : 's'} healthy.` };
|
|
216
|
+
}
|
|
217
|
+
function topologyCheck(deployment) {
|
|
218
|
+
if (!deployment)
|
|
219
|
+
return { status: 'skip', detail: 'No deployment described.' };
|
|
220
|
+
if (deployment.replicas <= 1)
|
|
221
|
+
return { status: 'ok', detail: 'One replica: in-process state is shared by everything.' };
|
|
222
|
+
const findings = [];
|
|
223
|
+
for (const [name, value] of Object.entries(deployment.stores ?? {})) {
|
|
224
|
+
const kind = value?.constructor?.name;
|
|
225
|
+
const known = kind ? PROCESS_LOCAL[kind] : undefined;
|
|
226
|
+
if (known)
|
|
227
|
+
findings.push({ status: known.status, text: `${name} is a ${kind}: ${known.why}` });
|
|
228
|
+
}
|
|
229
|
+
if (findings.length === 0) {
|
|
230
|
+
return { status: 'ok', detail: `${deployment.replicas} replicas, and nothing keeps its state in one process.` };
|
|
231
|
+
}
|
|
232
|
+
return {
|
|
233
|
+
status: findings.some((finding) => finding.status === 'fail') ? 'fail' : 'warn',
|
|
234
|
+
detail: `${deployment.replicas} replicas: ${findings.map((finding) => finding.text).join('; ')}.`,
|
|
235
|
+
hint: 'Use the Redis, Postgres, or SQLite adapter for each, or run one replica.',
|
|
236
|
+
};
|
|
237
|
+
}
|