@juspay/neurolink 12.18.0 → 12.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -2
- package/README.md +277 -75
- package/dist/browser/neurolink.min.js +396 -396
- package/dist/cli/commands/setup.js +2 -1
- package/dist/cli/factories/commandFactory.d.ts +22 -2
- package/dist/cli/factories/commandFactory.js +14 -2
- package/dist/cli/loop/optionsSchema.js +4 -0
- package/dist/cli/utils/classifierRouterFlags.js +39 -1
- package/dist/constants/enums.d.ts +13 -0
- package/dist/constants/enums.js +14 -0
- package/dist/context/budgetChecker.d.ts +3 -1
- package/dist/context/budgetChecker.js +20 -4
- package/dist/context/contextCompactor.d.ts +43 -3
- package/dist/context/contextCompactor.js +157 -41
- package/dist/context/contextDecision.d.ts +56 -0
- package/dist/context/contextDecision.js +234 -0
- package/dist/core/baseProvider.d.ts +27 -1
- package/dist/core/baseProvider.js +39 -0
- package/dist/core/toolRouting.d.ts +8 -0
- package/dist/core/toolRouting.js +52 -1
- package/dist/core/toolRoutingDecision.d.ts +35 -0
- package/dist/core/toolRoutingDecision.js +194 -0
- package/dist/factories/providerDescriptors.d.ts +15 -0
- package/dist/factories/providerDescriptors.js +57 -1
- package/dist/factories/providerRegistry.js +9 -1
- package/dist/index.d.ts +57 -1
- package/dist/index.js +60 -1
- package/dist/models/manifestRegistry.js +2 -0
- package/dist/models/manifests/typesafe.d.ts +16 -0
- package/dist/models/manifests/typesafe.js +42 -0
- package/dist/neurolink.d.ts +57 -1
- package/dist/neurolink.js +254 -10
- package/dist/observability/exporters/laminarExporter.js +3 -0
- package/dist/observability/exporters/posthogExporter.js +3 -0
- package/dist/observability/utils/spanSerializer.js +4 -0
- package/dist/providers/catalog/schema.d.ts +1 -1
- package/dist/providers/typesafe.d.ts +112 -0
- package/dist/providers/typesafe.js +654 -0
- package/dist/proxy/proxyHistoryTruncation.js +15 -10
- package/dist/rag/pipeline/RAGPipeline.d.ts +3 -1
- package/dist/rag/pipeline/RAGPipeline.js +65 -4
- package/dist/rag/retrieval/searchDecision.d.ts +27 -0
- package/dist/rag/retrieval/searchDecision.js +124 -0
- package/dist/routing/classifierRouter.d.ts +74 -2
- package/dist/routing/classifierRouter.js +232 -32
- package/dist/routing/classifierStrategies.d.ts +33 -1
- package/dist/routing/classifierStrategies.js +237 -0
- package/dist/routing/index.d.ts +1 -1
- package/dist/routing/index.js +1 -1
- package/dist/routing/modelCatalog.d.ts +79 -0
- package/dist/routing/modelCatalog.js +382 -0
- package/dist/telemetry/tracers.d.ts +1 -0
- package/dist/telemetry/tracers.js +1 -0
- package/dist/types/classifierRouter.d.ts +160 -4
- package/dist/types/cli.d.ts +17 -1
- package/dist/types/config.d.ts +12 -0
- package/dist/types/context.d.ts +55 -1
- package/dist/types/decision.d.ts +200 -0
- package/dist/types/decision.js +21 -0
- package/dist/types/generate.d.ts +20 -0
- package/dist/types/index.d.ts +1 -0
- package/dist/types/index.js +2 -0
- package/dist/types/providers.d.ts +59 -1
- package/dist/types/providers.js +5 -0
- package/dist/types/rag.d.ts +48 -0
- package/dist/types/span.d.ts +11 -0
- package/dist/types/span.js +11 -0
- package/dist/types/stream.d.ts +13 -0
- package/dist/types/toolRouting.d.ts +56 -0
- package/dist/utils/decisionAnswers.d.ts +56 -0
- package/dist/utils/decisionAnswers.js +95 -0
- package/dist/utils/modelChoices.js +13 -1
- package/dist/utils/pricing.js +11 -0
- package/dist/utils/providerConfig.d.ts +4 -0
- package/dist/utils/providerConfig.js +17 -0
- package/docs-site/static/search-index.json +1 -1
- package/package.json +16 -3
package/dist/neurolink.js
CHANGED
|
@@ -34,6 +34,7 @@ import { buildToolRoutingCatalog, buildRoutingQueryFromHistory, resolveToolRouti
|
|
|
34
34
|
import { ToolRoutingCache } from "./core/toolRoutingCache.js";
|
|
35
35
|
import { DEFAULT_RECENT_TURNS, KnowledgeGroundingEngine, } from "./knowledge/index.js";
|
|
36
36
|
import { AIProviderFactory } from "./core/factory.js";
|
|
37
|
+
import { resolveDefaultDecisionProvider } from "./factories/providerDescriptors.js";
|
|
37
38
|
import { resolveRequestKind } from "./core/resolveRequestKind.js";
|
|
38
39
|
import { createToolEventPayload } from "./core/toolEvents.js";
|
|
39
40
|
import { ProviderFactory } from "./factories/providerFactory.js";
|
|
@@ -66,6 +67,9 @@ import { getMetricsAggregator, MetricsAggregator, } from "./observability/metric
|
|
|
66
67
|
import { AnalyticsService, calculateAdvancedCost, parseAnalyticsQualityScore, } from "./analytics/index.js";
|
|
67
68
|
import { SpanStatus, SpanType, CircuitBreakerOpenError, ConversationMemoryError, ModelAccessDeniedError, } from "./types/index.js";
|
|
68
69
|
import { SpanSerializer } from "./observability/utils/spanSerializer.js";
|
|
70
|
+
import { withSpan } from "./telemetry/withSpan.js";
|
|
71
|
+
import { getActiveTraceContext } from "./telemetry/traceContext.js";
|
|
72
|
+
import { calculateCost } from "./utils/pricing.js";
|
|
69
73
|
import { flushOpenTelemetry, getLangfuseContext, getLangfuseHealthStatus, initializeOpenTelemetry, isOpenTelemetryInitialized, runWithCurrentLangfuseContext, setLangfuseContext, shutdownOpenTelemetry, stampGuestRescueIdentity, } from "./services/server/ai/observability/instrumentation.js";
|
|
70
74
|
import { clearChecklistState, createChecklistTools, getChecklistState, resolveChecklistSessionId, } from "./agent/taskChecklist.js";
|
|
71
75
|
import { cancelDelegates as cancelBackgroundDelegates, collectDelegates as collectBackgroundDelegates, configureDelegation, createDelegationTools, spawnDelegate as spawnBackgroundDelegate, } from "./agent/backgroundDelegation.js";
|
|
@@ -485,6 +489,10 @@ export class NeuroLink {
|
|
|
485
489
|
// ClassifierRouter: opt-in "classify → pick model + tools" pre-call router.
|
|
486
490
|
// Built from config.classifierRouter; null when not configured.
|
|
487
491
|
classifierRouter;
|
|
492
|
+
// Tuning for the decision-driven relevance stage of context compaction.
|
|
493
|
+
// Undefined = the stage's own defaults; the stage itself is gated on a
|
|
494
|
+
// decision provider being configured, not on this.
|
|
495
|
+
contextRelevanceOptions;
|
|
488
496
|
/**
|
|
489
497
|
* Merge instance-level credentials with per-call credentials.
|
|
490
498
|
*
|
|
@@ -943,6 +951,10 @@ export class NeuroLink {
|
|
|
943
951
|
this.modelPool = config?.modelPool ? new ModelPool(config.modelPool) : null;
|
|
944
952
|
// RequestRouter: store the host-supplied function; null when not configured.
|
|
945
953
|
this.requestRouter = config?.requestRouter ?? null;
|
|
954
|
+
// Tuning for the relevance stage of compaction. Storing it (rather than
|
|
955
|
+
// reading config at each call site) keeps compactor construction uniform
|
|
956
|
+
// across the five places that build one.
|
|
957
|
+
this.contextRelevanceOptions = config?.contextRelevance;
|
|
946
958
|
// ClassifierRouter: opt-in. The LLM strategy reuses this instance's
|
|
947
959
|
// generate() (marked so it never recursively re-routes). Fails open.
|
|
948
960
|
this.classifierRouter = config?.classifierRouter?.enabled
|
|
@@ -950,6 +962,10 @@ export class NeuroLink {
|
|
|
950
962
|
generate: (genOptions) => this.generate({
|
|
951
963
|
...genOptions,
|
|
952
964
|
}),
|
|
965
|
+
// Fail-open by construction: tryDecide returns null rather than
|
|
966
|
+
// throwing, so an absent or broken decision provider leaves routing
|
|
967
|
+
// exactly as it was.
|
|
968
|
+
decide: (decideOptions) => this.tryDecide(decideOptions),
|
|
953
969
|
logger: {
|
|
954
970
|
debug: (message, meta) => logger.debug(message, meta),
|
|
955
971
|
warn: (message, meta) => logger.warn(message, meta),
|
|
@@ -3955,6 +3971,19 @@ Current user's request: ${currentInput}`;
|
|
|
3955
3971
|
});
|
|
3956
3972
|
}
|
|
3957
3973
|
}
|
|
3974
|
+
/**
|
|
3975
|
+
* Dependencies handed to every `ContextCompactor`.
|
|
3976
|
+
*
|
|
3977
|
+
* `tryDecide` returns null whenever no decision provider is configured or
|
|
3978
|
+
* the call fails, so passing this unconditionally is safe: the compactor
|
|
3979
|
+
* runs its previous pipeline unchanged and pays nothing for the option.
|
|
3980
|
+
*/
|
|
3981
|
+
contextCompactorDeps() {
|
|
3982
|
+
return {
|
|
3983
|
+
decide: (decisionOptions) => this.tryDecide(decisionOptions),
|
|
3984
|
+
relevance: this.contextRelevanceOptions,
|
|
3985
|
+
};
|
|
3986
|
+
}
|
|
3958
3987
|
/**
|
|
3959
3988
|
* Applies the host-configured `classifierRouter` to `options` in place.
|
|
3960
3989
|
*
|
|
@@ -4029,12 +4058,47 @@ Current user's request: ${currentInput}`;
|
|
|
4029
4058
|
: [...allow];
|
|
4030
4059
|
}
|
|
4031
4060
|
}
|
|
4061
|
+
// Context budget: only ever narrowed, and only when the caller left it
|
|
4062
|
+
// unset. A caller that pinned a threshold has made a statement about
|
|
4063
|
+
// their own traffic that a per-request guess should not overrule.
|
|
4064
|
+
if (decision.compactionThreshold !== undefined &&
|
|
4065
|
+
options.compactionThreshold === undefined) {
|
|
4066
|
+
options.compactionThreshold = decision.compactionThreshold;
|
|
4067
|
+
logger.debug("[NeuroLink] classifierRouter context budget", {
|
|
4068
|
+
compactionThreshold: decision.compactionThreshold,
|
|
4069
|
+
contextScope: decision.contextScope,
|
|
4070
|
+
});
|
|
4071
|
+
}
|
|
4032
4072
|
// Mark routed so orchestration/requestRouter stand down this turn.
|
|
4033
4073
|
if (decision.provider || decision.model) {
|
|
4034
4074
|
const ctx = opt.context ?? {};
|
|
4035
4075
|
ctx.__classifierRouted = true;
|
|
4036
4076
|
opt.context = ctx;
|
|
4037
4077
|
}
|
|
4078
|
+
// Telemetry: a debug log is not an observability surface. Routing
|
|
4079
|
+
// silently changes which model — and therefore which bill — serves a
|
|
4080
|
+
// request, so the decision has to be visible on the trace next to the
|
|
4081
|
+
// generation it produced.
|
|
4082
|
+
try {
|
|
4083
|
+
const activeSpan = trace.getActiveSpan();
|
|
4084
|
+
if (activeSpan) {
|
|
4085
|
+
activeSpan.setAttribute("classifier_routing.difficulty", decision.difficulty ?? "unknown");
|
|
4086
|
+
activeSpan.setAttribute("classifier_routing.provider", options.provider ?? "");
|
|
4087
|
+
activeSpan.setAttribute("classifier_routing.model", options.model ?? "");
|
|
4088
|
+
if (decision.contextScope) {
|
|
4089
|
+
activeSpan.setAttribute("classifier_routing.context_scope", decision.contextScope);
|
|
4090
|
+
}
|
|
4091
|
+
if (decision.compactionThreshold !== undefined) {
|
|
4092
|
+
activeSpan.setAttribute("classifier_routing.compaction_threshold", decision.compactionThreshold);
|
|
4093
|
+
}
|
|
4094
|
+
if (decision.reason) {
|
|
4095
|
+
activeSpan.setAttribute("classifier_routing.reason", decision.reason);
|
|
4096
|
+
}
|
|
4097
|
+
}
|
|
4098
|
+
}
|
|
4099
|
+
catch {
|
|
4100
|
+
// Telemetry must never affect routing behaviour.
|
|
4101
|
+
}
|
|
4038
4102
|
if (decision.reason) {
|
|
4039
4103
|
logger.debug("[NeuroLink] classifierRouter decision", {
|
|
4040
4104
|
reason: decision.reason,
|
|
@@ -5142,7 +5206,7 @@ Current user's request: ${currentInput}`;
|
|
|
5142
5206
|
enableDeduplicate: true,
|
|
5143
5207
|
enableTruncate: true,
|
|
5144
5208
|
truncationFraction: fraction,
|
|
5145
|
-
});
|
|
5209
|
+
}, this.contextCompactorDeps());
|
|
5146
5210
|
const compactionResult = await compactor.compact(originalMessages, compactionTarget, undefined, options.context?.requestId);
|
|
5147
5211
|
if (!compactionResult.compacted) {
|
|
5148
5212
|
continue;
|
|
@@ -5715,6 +5779,7 @@ Current user's request: ${currentInput}`;
|
|
|
5715
5779
|
conversationMessages: conversationMessages,
|
|
5716
5780
|
currentPrompt: options.prompt,
|
|
5717
5781
|
toolDefinitions: availableTools,
|
|
5782
|
+
compactionThreshold: options.compactionThreshold,
|
|
5718
5783
|
});
|
|
5719
5784
|
logger.info("[TokenBudget] Token breakdown", {
|
|
5720
5785
|
requestId,
|
|
@@ -5759,7 +5824,7 @@ Current user's request: ${currentInput}`;
|
|
|
5759
5824
|
availableTools,
|
|
5760
5825
|
conversationMessages,
|
|
5761
5826
|
availableInputTokens: budgetResult.availableInputTokens,
|
|
5762
|
-
historyBudget: resolveHistoryBudget(budgetResult),
|
|
5827
|
+
historyBudget: resolveHistoryBudget(budgetResult, options.compactionThreshold),
|
|
5763
5828
|
usageRatio: budgetResult.usageRatio,
|
|
5764
5829
|
estimatedInputTokens: budgetResult.estimatedInputTokens,
|
|
5765
5830
|
compactionSessionId,
|
|
@@ -5777,7 +5842,7 @@ Current user's request: ${currentInput}`;
|
|
|
5777
5842
|
summarizationProvider: this.conversationMemoryConfig?.conversationMemory
|
|
5778
5843
|
?.summarizationProvider,
|
|
5779
5844
|
summarizationModel: this.conversationMemoryConfig?.conversationMemory?.summarizationModel,
|
|
5780
|
-
});
|
|
5845
|
+
}, this.contextCompactorDeps());
|
|
5781
5846
|
// Fixed overhead (system + prompt + tools + files) already exceeds the
|
|
5782
5847
|
// window — no amount of history compaction can fit this request, and
|
|
5783
5848
|
// compacting to an empty history would only hide the real cause.
|
|
@@ -5792,7 +5857,7 @@ Current user's request: ${currentInput}`;
|
|
|
5792
5857
|
breakdown: {},
|
|
5793
5858
|
});
|
|
5794
5859
|
}
|
|
5795
|
-
const compactionResult = await compactor.compact(conversationMessages, historyBudget, this.conversationMemoryConfig?.conversationMemory, requestId);
|
|
5860
|
+
const compactionResult = await compactor.compact(conversationMessages, historyBudget, this.conversationMemoryConfig?.conversationMemory, requestId, { currentRequest: options.prompt });
|
|
5796
5861
|
let compactedMessages = conversationMessages;
|
|
5797
5862
|
if (compactionResult.compacted) {
|
|
5798
5863
|
const repairedResult = repairToolPairs(compactionResult.messages);
|
|
@@ -6169,6 +6234,7 @@ Current user's request: ${currentInput}`;
|
|
|
6169
6234
|
toolDefinitions: options.tools
|
|
6170
6235
|
? Object.values(options.tools)
|
|
6171
6236
|
: undefined,
|
|
6237
|
+
compactionThreshold: options.compactionThreshold,
|
|
6172
6238
|
});
|
|
6173
6239
|
const dpgMessageCount = conversationMessages?.length || 0;
|
|
6174
6240
|
const dpgCompactionSessionId = this.getCompactionSessionId(options);
|
|
@@ -6311,8 +6377,8 @@ Current user's request: ${currentInput}`;
|
|
|
6311
6377
|
?.summarizationProvider,
|
|
6312
6378
|
summarizationModel: this.conversationMemoryConfig?.conversationMemory
|
|
6313
6379
|
?.summarizationModel,
|
|
6314
|
-
});
|
|
6315
|
-
const compactionResult = await compactor.compact(conversationMessages, resolveHistoryBudget(budgetCheck), this.conversationMemoryConfig?.conversationMemory, options.context?.requestId);
|
|
6380
|
+
}, this.contextCompactorDeps());
|
|
6381
|
+
const compactionResult = await compactor.compact(conversationMessages, resolveHistoryBudget(budgetCheck, options.compactionThreshold), this.conversationMemoryConfig?.conversationMemory, options.context?.requestId, { currentRequest: options.prompt });
|
|
6316
6382
|
if (compactionResult.compacted) {
|
|
6317
6383
|
const repairedResult = repairToolPairs(compactionResult.messages);
|
|
6318
6384
|
conversationMessages = repairedResult.messages;
|
|
@@ -7158,6 +7224,14 @@ Current user's request: ${currentInput}`;
|
|
|
7158
7224
|
if (decision.granularity !== undefined) {
|
|
7159
7225
|
activeSpan.setAttribute("tool_routing.granularity", decision.granularity);
|
|
7160
7226
|
}
|
|
7227
|
+
// Which router actually decided. Three strategies can produce an
|
|
7228
|
+
// "applied" outcome and they have wildly different cost and
|
|
7229
|
+
// latency profiles, so an outcome without a strategy cannot be
|
|
7230
|
+
// acted on — and a decision router that silently stopped firing
|
|
7231
|
+
// looks identical to one that was never configured.
|
|
7232
|
+
if (decision.strategy !== undefined) {
|
|
7233
|
+
activeSpan.setAttribute("tool_routing.strategy", decision.strategy);
|
|
7234
|
+
}
|
|
7161
7235
|
}
|
|
7162
7236
|
catch {
|
|
7163
7237
|
// Telemetry must never affect routing behaviour.
|
|
@@ -7288,6 +7362,14 @@ Current user's request: ${currentInput}`;
|
|
|
7288
7362
|
...generateOptions,
|
|
7289
7363
|
abortSignal: options.abortSignal,
|
|
7290
7364
|
}),
|
|
7365
|
+
// Calibrated per-server routing when a decision provider is
|
|
7366
|
+
// configured. tryDecide returns null without one, so the resolver
|
|
7367
|
+
// falls straight through to the generative router as before.
|
|
7368
|
+
decideFn: (decisionOptions) => this.tryDecide({
|
|
7369
|
+
...decisionOptions,
|
|
7370
|
+
signal: options.abortSignal,
|
|
7371
|
+
}),
|
|
7372
|
+
decisionMinDropConfidence: routingConfig.minDropConfidence,
|
|
7291
7373
|
emitDecision: captureDecision,
|
|
7292
7374
|
// L2 / ITEM D — only populated when embedding is configured.
|
|
7293
7375
|
embedFn: routingEmbedFn,
|
|
@@ -8570,6 +8652,7 @@ Current user's request: ${currentInput}`;
|
|
|
8570
8652
|
conversationMessages: conversationMessages,
|
|
8571
8653
|
currentPrompt: options.input.text,
|
|
8572
8654
|
toolDefinitions: availableTools,
|
|
8655
|
+
compactionThreshold: options.compactionThreshold,
|
|
8573
8656
|
});
|
|
8574
8657
|
const streamMessageCount = conversationMessages?.length || 0;
|
|
8575
8658
|
const streamCompactionSessionId = this.getCompactionSessionId(options);
|
|
@@ -8617,9 +8700,10 @@ Current user's request: ${currentInput}`;
|
|
|
8617
8700
|
provider: providerName,
|
|
8618
8701
|
summarizationProvider: this.conversationMemoryConfig?.conversationMemory
|
|
8619
8702
|
?.summarizationProvider,
|
|
8620
|
-
summarizationModel: this.conversationMemoryConfig?.conversationMemory
|
|
8621
|
-
|
|
8622
|
-
|
|
8703
|
+
summarizationModel: this.conversationMemoryConfig?.conversationMemory
|
|
8704
|
+
?.summarizationModel,
|
|
8705
|
+
}, this.contextCompactorDeps());
|
|
8706
|
+
const compactionResult = await compactor.compact(conversationMessages, resolveHistoryBudget(streamBudget, options.compactionThreshold), this.conversationMemoryConfig?.conversationMemory, options.context?.requestId, { currentRequest: options.input.text });
|
|
8623
8707
|
if (compactionResult.compacted) {
|
|
8624
8708
|
const repairedResult = repairToolPairs(compactionResult.messages);
|
|
8625
8709
|
conversationMessages = repairedResult.messages;
|
|
@@ -12563,6 +12647,166 @@ Current user's request: ${currentInput}`;
|
|
|
12563
12647
|
logger.debug(`[NeuroLink] Created evaluation pipeline: ${config.name ?? "custom"}`);
|
|
12564
12648
|
return pipeline;
|
|
12565
12649
|
}
|
|
12650
|
+
/**
|
|
12651
|
+
* Run the `decide` inference type: evaluate a `state` against a batch of
|
|
12652
|
+
* named, typed questions and get one typed answer each.
|
|
12653
|
+
*
|
|
12654
|
+
* This is NOT {@link NeuroLink.evaluate}, which scores an already-generated
|
|
12655
|
+
* response with RAGAS scorers. `decide` calls a decision model — one that
|
|
12656
|
+
* emits no text at all and returns calibrated, typed judgements your code
|
|
12657
|
+
* can branch on.
|
|
12658
|
+
*
|
|
12659
|
+
* **Batch, never fan out.** On a decision model, latency is flat in
|
|
12660
|
+
* question count (measured: 1 question 393ms, 400 questions 465ms) while
|
|
12661
|
+
* concurrent requests queue. Put every question you might need into one
|
|
12662
|
+
* call — speculative questions are nearly free, a second round trip is not.
|
|
12663
|
+
*
|
|
12664
|
+
* A `choice` answer carries the full probability distribution, so one
|
|
12665
|
+
* question over N options also ranks all N.
|
|
12666
|
+
*
|
|
12667
|
+
* @example
|
|
12668
|
+
* ```typescript
|
|
12669
|
+
* const result = await neurolink.decide({
|
|
12670
|
+
* state: ticketText,
|
|
12671
|
+
* questions: {
|
|
12672
|
+
* team: {
|
|
12673
|
+
* type: "choice",
|
|
12674
|
+
* instructions: "Which team should handle this?",
|
|
12675
|
+
* criteria: { billing: "Payments", technical: "Bugs", sales: "Pricing" },
|
|
12676
|
+
* },
|
|
12677
|
+
* urgent: { type: "boolean", instructions: "Is this urgent?" },
|
|
12678
|
+
* },
|
|
12679
|
+
* });
|
|
12680
|
+
* const team = readDecisionChoice(result.answers, "team");
|
|
12681
|
+
* if (team && team.confidence > 0.7) { assign(team.choice); }
|
|
12682
|
+
* ```
|
|
12683
|
+
*
|
|
12684
|
+
* @throws when no decision provider is configured, or the call fails
|
|
12685
|
+
*/
|
|
12686
|
+
async decide(options) {
|
|
12687
|
+
const providerName = options.provider ?? resolveDefaultDecisionProvider();
|
|
12688
|
+
if (!providerName) {
|
|
12689
|
+
throw new Error("No decision provider is configured. Set TYPESAFE_API_KEY, or pass `provider` explicitly.");
|
|
12690
|
+
}
|
|
12691
|
+
const questionCount = Object.keys(options.questions).length;
|
|
12692
|
+
// Observability parity with generate()/stream(): an OTEL span for the
|
|
12693
|
+
// trace view, a serialized span for the in-process metrics aggregator,
|
|
12694
|
+
// and a cost figure. Without these a decision model is invisible — and
|
|
12695
|
+
// "invisible" is the failure mode that matters here, because every
|
|
12696
|
+
// internal consumer is fail-open, so a decision path that has silently
|
|
12697
|
+
// stopped working looks exactly like one that was never configured.
|
|
12698
|
+
return withSpan({
|
|
12699
|
+
name: `gen_ai.${providerName}.decide`,
|
|
12700
|
+
tracer: tracers.decision,
|
|
12701
|
+
attributes: {
|
|
12702
|
+
"gen_ai.system": providerName,
|
|
12703
|
+
"gen_ai.operation.name": "decide",
|
|
12704
|
+
"ai.provider": providerName,
|
|
12705
|
+
"decision.question_count": questionCount,
|
|
12706
|
+
},
|
|
12707
|
+
}, async (otelSpan) => {
|
|
12708
|
+
const { traceId, parentSpanId } = getActiveTraceContext();
|
|
12709
|
+
const startedAt = Date.now();
|
|
12710
|
+
let span = SpanSerializer.createSpan(SpanType.MODEL_DECISION, `decision.${providerName}`, {
|
|
12711
|
+
"ai.provider": providerName,
|
|
12712
|
+
"ai.model": options.model ?? "",
|
|
12713
|
+
"decision.question_count": questionCount,
|
|
12714
|
+
}, parentSpanId, traceId);
|
|
12715
|
+
try {
|
|
12716
|
+
// enableMCP: false — a decision model has no tools and never will.
|
|
12717
|
+
const provider = await AIProviderFactory.createProvider(providerName, options.model, false, this, undefined, this.resolveCredentials(options.credentials));
|
|
12718
|
+
if (typeof provider.decide !== "function") {
|
|
12719
|
+
throw new Error(`Provider "${providerName}" does not serve the decide inference type.`);
|
|
12720
|
+
}
|
|
12721
|
+
const result = await provider.decide({
|
|
12722
|
+
state: options.state,
|
|
12723
|
+
questions: options.questions,
|
|
12724
|
+
model: options.model,
|
|
12725
|
+
signal: options.signal,
|
|
12726
|
+
timeoutMs: options.timeoutMs,
|
|
12727
|
+
});
|
|
12728
|
+
const answerCount = Object.keys(result.answers).length;
|
|
12729
|
+
const cost = calculateCost(providerName, result.model, {
|
|
12730
|
+
input: result.usage.inputTokens,
|
|
12731
|
+
output: result.usage.outputTokens,
|
|
12732
|
+
total: result.usage.inputTokens + result.usage.outputTokens,
|
|
12733
|
+
});
|
|
12734
|
+
otelSpan.setAttribute("gen_ai.response.model", result.model);
|
|
12735
|
+
otelSpan.setAttribute("gen_ai.usage.input_tokens", result.usage.inputTokens);
|
|
12736
|
+
otelSpan.setAttribute("gen_ai.usage.output_tokens", result.usage.outputTokens);
|
|
12737
|
+
otelSpan.setAttribute("decision.answer_count", answerCount);
|
|
12738
|
+
otelSpan.setAttribute("ai.cost.total", cost);
|
|
12739
|
+
if (result.upstreamMs !== undefined) {
|
|
12740
|
+
otelSpan.setAttribute("decision.upstream_ms", result.upstreamMs);
|
|
12741
|
+
}
|
|
12742
|
+
if (result.requestId) {
|
|
12743
|
+
otelSpan.setAttribute("gen_ai.response.id", result.requestId);
|
|
12744
|
+
}
|
|
12745
|
+
span.durationMs = Date.now() - startedAt;
|
|
12746
|
+
span = SpanSerializer.updateAttributes(span, {
|
|
12747
|
+
// The key names the cost aggregator reads. `ai.model` is
|
|
12748
|
+
// overwritten with the RESOLVED model (e.g. "jev-1.13.0"), not
|
|
12749
|
+
// the alias that was sent, so spend is attributed to what
|
|
12750
|
+
// actually ran.
|
|
12751
|
+
"ai.model": result.model,
|
|
12752
|
+
"ai.tokens.input": result.usage.inputTokens,
|
|
12753
|
+
"ai.tokens.output": result.usage.outputTokens,
|
|
12754
|
+
"ai.cost.input": cost,
|
|
12755
|
+
"ai.cost.output": 0,
|
|
12756
|
+
"ai.cost.total": cost,
|
|
12757
|
+
"decision.answer_count": answerCount,
|
|
12758
|
+
"decision.latency_ms": result.latencyMs,
|
|
12759
|
+
...(result.upstreamMs !== undefined
|
|
12760
|
+
? { "decision.upstream_ms": result.upstreamMs }
|
|
12761
|
+
: {}),
|
|
12762
|
+
});
|
|
12763
|
+
// BOTH aggregators, exactly as the generate path does. The
|
|
12764
|
+
// instance one is what `neurolink.getMetrics()` / `getSpans()`
|
|
12765
|
+
// read; the global one is what the exporters drain. Recording to
|
|
12766
|
+
// only the global one would make a decision invisible to the very
|
|
12767
|
+
// API a caller would use to check whether it ran.
|
|
12768
|
+
const ended = SpanSerializer.endSpan(span, SpanStatus.OK);
|
|
12769
|
+
this.metricsAggregator.recordSpan(ended);
|
|
12770
|
+
getMetricsAggregator().recordSpan(ended);
|
|
12771
|
+
return result;
|
|
12772
|
+
}
|
|
12773
|
+
catch (error) {
|
|
12774
|
+
span.durationMs = Date.now() - startedAt;
|
|
12775
|
+
const ended = SpanSerializer.endSpan(span, SpanStatus.ERROR, error instanceof Error ? error.message : String(error));
|
|
12776
|
+
this.metricsAggregator.recordSpan(ended);
|
|
12777
|
+
getMetricsAggregator().recordSpan(ended);
|
|
12778
|
+
throw error;
|
|
12779
|
+
}
|
|
12780
|
+
});
|
|
12781
|
+
}
|
|
12782
|
+
/**
|
|
12783
|
+
* Fail-open {@link NeuroLink.decide}: returns null on any failure instead
|
|
12784
|
+
* of throwing.
|
|
12785
|
+
*
|
|
12786
|
+
* This is the contract every internal consumer relies on. A decision model
|
|
12787
|
+
* that is unconfigured, slow, rate-limited or down must never change
|
|
12788
|
+
* NeuroLink's observable behaviour — the caller falls back to whatever it
|
|
12789
|
+
* did before the decision was available.
|
|
12790
|
+
*/
|
|
12791
|
+
async tryDecide(options) {
|
|
12792
|
+
try {
|
|
12793
|
+
const result = await this.decide(options);
|
|
12794
|
+
if (logger.shouldLog("debug")) {
|
|
12795
|
+
logger.debug(`decide: ${Object.keys(result.answers).length} answers in ${result.latencyMs}ms` +
|
|
12796
|
+
(result.upstreamMs ? ` (upstream ${result.upstreamMs}ms)` : ""), {
|
|
12797
|
+
provider: result.provider,
|
|
12798
|
+
model: result.model,
|
|
12799
|
+
requestId: result.requestId,
|
|
12800
|
+
usage: result.usage,
|
|
12801
|
+
});
|
|
12802
|
+
}
|
|
12803
|
+
return result;
|
|
12804
|
+
}
|
|
12805
|
+
catch (error) {
|
|
12806
|
+
logger.debug(`decide unavailable: ${error instanceof Error ? error.message : String(error)} — falling back`);
|
|
12807
|
+
return null;
|
|
12808
|
+
}
|
|
12809
|
+
}
|
|
12566
12810
|
/**
|
|
12567
12811
|
* Evaluate an AI response using the specified pipeline or scorers.
|
|
12568
12812
|
* This is a convenience method that creates a pipeline and executes it in one call.
|
|
@@ -13796,7 +14040,7 @@ Current user's request: ${currentInput}`;
|
|
|
13796
14040
|
?.summarizationProvider,
|
|
13797
14041
|
summarizationModel: config?.summarizationModel ??
|
|
13798
14042
|
this.conversationMemoryConfig?.conversationMemory?.summarizationModel,
|
|
13799
|
-
});
|
|
14043
|
+
}, this.contextCompactorDeps());
|
|
13800
14044
|
// Use actual context window to determine target, not arbitrary heuristic
|
|
13801
14045
|
const budgetInfo = checkContextBudget({
|
|
13802
14046
|
provider: config?.provider || "openai",
|
|
@@ -222,6 +222,9 @@ export class LaminarExporter extends BaseExporter {
|
|
|
222
222
|
[SpanType.WORKFLOW_STEP]: "workflow",
|
|
223
223
|
[SpanType.TOOL_CALL]: "tool",
|
|
224
224
|
[SpanType.MODEL_GENERATION]: "llm",
|
|
225
|
+
// Laminar has no decision primitive; "llm" is the closest bucket that
|
|
226
|
+
// keeps it visible on a model-spend view.
|
|
227
|
+
[SpanType.MODEL_DECISION]: "llm",
|
|
225
228
|
[SpanType.EMBEDDING]: "embedding",
|
|
226
229
|
[SpanType.RETRIEVAL]: "retrieval",
|
|
227
230
|
[SpanType.MEMORY]: "memory",
|
|
@@ -205,6 +205,9 @@ export class PostHogExporter extends BaseExporter {
|
|
|
205
205
|
[SpanType.WORKFLOW_STEP]: "ai_workflow_step",
|
|
206
206
|
[SpanType.TOOL_CALL]: "ai_tool_call",
|
|
207
207
|
[SpanType.MODEL_GENERATION]: "ai_generation",
|
|
208
|
+
// A distinct event name: a decision is not a generation, and merging
|
|
209
|
+
// them would silently inflate generation funnels in PostHog.
|
|
210
|
+
[SpanType.MODEL_DECISION]: "ai_decision",
|
|
208
211
|
[SpanType.EMBEDDING]: "ai_embedding",
|
|
209
212
|
[SpanType.RETRIEVAL]: "ai_retrieval",
|
|
210
213
|
[SpanType.MEMORY]: "ai_memory_operation",
|
|
@@ -223,6 +223,10 @@ export class SpanSerializer {
|
|
|
223
223
|
[SpanType.WORKFLOW_STEP]: "chain",
|
|
224
224
|
[SpanType.TOOL_CALL]: "tool",
|
|
225
225
|
[SpanType.MODEL_GENERATION]: "llm",
|
|
226
|
+
// Langfuse observation kinds are a closed set with no decision member;
|
|
227
|
+
// "llm" keeps the call on the model-cost views, which is where a
|
|
228
|
+
// reader looking for spend will go find it.
|
|
229
|
+
[SpanType.MODEL_DECISION]: "llm",
|
|
226
230
|
[SpanType.EMBEDDING]: "embedding",
|
|
227
231
|
[SpanType.RETRIEVAL]: "retriever",
|
|
228
232
|
[SpanType.MEMORY]: "chain",
|
|
@@ -73,8 +73,8 @@ export declare const providerCatalogJsonSchema: z.ZodObject<{
|
|
|
73
73
|
class: z.ZodEnum<{
|
|
74
74
|
network: "network";
|
|
75
75
|
provider: "provider";
|
|
76
|
-
"invalid-model": "invalid-model";
|
|
77
76
|
authentication: "authentication";
|
|
77
|
+
"invalid-model": "invalid-model";
|
|
78
78
|
"rate-limit": "rate-limit";
|
|
79
79
|
}>;
|
|
80
80
|
message: z.ZodString;
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import type { AIProviderName } from "../constants/enums.js";
|
|
2
|
+
import { BaseProvider } from "../core/baseProvider.js";
|
|
3
|
+
import type { DecisionModelCard, DecisionRequest, DecisionResult, LanguageModel, NeurolinkCredentials, StreamOptions, StreamResult, ValidationSchema } from "../types/index.js";
|
|
4
|
+
/**
|
|
5
|
+
* The binding limit in practice: `state` plus the SINGLE LONGEST question.
|
|
6
|
+
* Measured by bisection at single-character resolution — 33,002 reported
|
|
7
|
+
* input tokens accepted, 33,003 not.
|
|
8
|
+
*/
|
|
9
|
+
export declare const TYPESAFE_MAX_STATE_TOKENS = 33000;
|
|
10
|
+
/**
|
|
11
|
+
* The separate, larger ceiling on `state` plus ALL questions combined. A
|
|
12
|
+
* request can carry a near-ceiling state *and* 400 extra questions (37,679
|
|
13
|
+
* tokens total, verified) — questions do not compete with state for the
|
|
14
|
+
* 33K budget, only for this one.
|
|
15
|
+
*/
|
|
16
|
+
export declare const TYPESAFE_MAX_REQUEST_TOKENS = 64000;
|
|
17
|
+
/**
|
|
18
|
+
* TypeSafe Provider — the `decide` inference type only.
|
|
19
|
+
*
|
|
20
|
+
* Jev is a "System One" model: it takes one `state` plus a map of typed
|
|
21
|
+
* questions and returns one typed answer each, evaluated in a single
|
|
22
|
+
* parallel pass. It emits no text, so `generate()` / `stream()` are not
|
|
23
|
+
* available and `getAISDKModel()` throws — the same shape Voyage and Jina
|
|
24
|
+
* already use for embedding-only providers.
|
|
25
|
+
*
|
|
26
|
+
* Its descriptor declares `inferenceKinds: ["decide"]`, which keeps it out
|
|
27
|
+
* of auto-select and the health sweep, so those throws should be unreachable
|
|
28
|
+
* in practice.
|
|
29
|
+
*
|
|
30
|
+
* Two measured properties shape the API: latency is flat in question count
|
|
31
|
+
* (1 question 393ms, 400 questions 465ms) while concurrent requests queue.
|
|
32
|
+
* Callers therefore batch every question into one `decide()` call.
|
|
33
|
+
*
|
|
34
|
+
* Two transports reach the same model. The direct one is TypeSafe's own API,
|
|
35
|
+
* billed against a TypeSafe key. The gateway one is Vercel's AI Gateway
|
|
36
|
+
* evaluation-model route, billed against a Vercel account — useful to a host
|
|
37
|
+
* that already has one and does not want a second vendor relationship. They
|
|
38
|
+
* differ on the wire (vocabulary, where the model is named, and whether
|
|
39
|
+
* `confidence` is reported at all) and not at all in `DecisionResult`; the
|
|
40
|
+
* one thing to know is that gateway confidence is DERIVED from the
|
|
41
|
+
* probability distribution rather than being the vendor's calibrated figure,
|
|
42
|
+
* so a threshold tuned on one transport should be re-checked on the other.
|
|
43
|
+
*
|
|
44
|
+
* @see https://docs.typesafe.ai/api
|
|
45
|
+
*/
|
|
46
|
+
export declare class TypeSafeProvider extends BaseProvider {
|
|
47
|
+
private readonly apiKey;
|
|
48
|
+
private readonly baseURL;
|
|
49
|
+
private readonly proxyFetch;
|
|
50
|
+
private readonly maxRetries;
|
|
51
|
+
/** Which wire this instance speaks. Resolved once, in the constructor. */
|
|
52
|
+
private readonly transport;
|
|
53
|
+
/**
|
|
54
|
+
* Set once the key has been rejected. A bad key cannot fix itself, and
|
|
55
|
+
* without this every later request would pay a full round trip (~265ms
|
|
56
|
+
* measured) to be told so again — a permanent tax on the hot path for a
|
|
57
|
+
* misconfiguration.
|
|
58
|
+
*/
|
|
59
|
+
private authFailed;
|
|
60
|
+
constructor(modelName?: string, sdk?: unknown, _region?: string, credentials?: NeurolinkCredentials["typesafe"]);
|
|
61
|
+
protected getProviderName(): AIProviderName;
|
|
62
|
+
protected getDefaultModel(): string;
|
|
63
|
+
protected getDefaultDecisionModel(): string | undefined;
|
|
64
|
+
supportsTools(): boolean;
|
|
65
|
+
/**
|
|
66
|
+
* Jev produces no text, so there is no chat model behind this provider.
|
|
67
|
+
* Unreachable while the descriptor declares `inferenceKinds: ["decide"]`;
|
|
68
|
+
* kept actionable for the case where something routes here anyway.
|
|
69
|
+
*/
|
|
70
|
+
protected getAISDKModel(): LanguageModel;
|
|
71
|
+
protected executeStream(_options: StreamOptions, _analysisSchema?: ValidationSchema): Promise<StreamResult>;
|
|
72
|
+
/**
|
|
73
|
+
* Must RETURN the error, never throw it (CLAUDE.md rule 6).
|
|
74
|
+
*/
|
|
75
|
+
protected formatProviderError(error: unknown): Error;
|
|
76
|
+
/**
|
|
77
|
+
* Evaluate a state against a batch of typed questions.
|
|
78
|
+
*
|
|
79
|
+
* Throws a {@link ProviderError} carrying a normalised {@link DecisionError}
|
|
80
|
+
* in `cause`. Internal hot paths should prefer `NeuroLink.tryDecide()`,
|
|
81
|
+
* which returns null instead.
|
|
82
|
+
*/
|
|
83
|
+
decide(request: DecisionRequest): Promise<DecisionResult>;
|
|
84
|
+
/** Where a decision request goes for this instance's transport. */
|
|
85
|
+
private endpoint;
|
|
86
|
+
/**
|
|
87
|
+
* Headers for this instance's transport.
|
|
88
|
+
*
|
|
89
|
+
* The gateway's three `ai-*` headers are all load-bearing:
|
|
90
|
+
* `ai-gateway-protocol-version` omitted fails the request outright rather
|
|
91
|
+
* than defaulting, `ai-model-id` is how the model is named at all (the body
|
|
92
|
+
* carries no `model`), and `ai-evaluation-model-specification-version`
|
|
93
|
+
* selects the request/response shape this provider parses.
|
|
94
|
+
*/
|
|
95
|
+
private headers;
|
|
96
|
+
/** Which transport this instance resolved to. Exposed for diagnostics. */
|
|
97
|
+
getDecisionTransport(): "direct" | "gateway";
|
|
98
|
+
/**
|
|
99
|
+
* The model aliases this key may send in `model`.
|
|
100
|
+
*
|
|
101
|
+
* Direct transport only: the gateway names the model in a header and
|
|
102
|
+
* publishes its own catalogue, so there is no equivalent listing behind
|
|
103
|
+
* this key.
|
|
104
|
+
*/
|
|
105
|
+
listDecisionModels(): Promise<DecisionModelCard[]>;
|
|
106
|
+
/**
|
|
107
|
+
* Wrap a normalised decision failure in the provider error type, keeping
|
|
108
|
+
* the structured detail reachable via `cause` for callers that branch on
|
|
109
|
+
* `kind` (authentication vs rate_limit vs max_tokens_exceeded).
|
|
110
|
+
*/
|
|
111
|
+
private decisionError;
|
|
112
|
+
}
|