@adaptic/utils 0.0.1014 → 0.0.1016
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +2823 -25
- package/dist/index.cjs.map +1 -1
- package/dist/index.mjs +2783 -25
- package/dist/index.mjs.map +1 -1
- package/dist/types/__tests__/llm/client/support/rejections.d.ts +21 -0
- package/dist/types/__tests__/llm/client/support/rejections.d.ts.map +1 -0
- package/dist/types/__tests__/llm/client/support/routes.d.ts +67 -0
- package/dist/types/__tests__/llm/client/support/routes.d.ts.map +1 -0
- package/dist/types/__tests__/llm/client/support/streams.d.ts +74 -0
- package/dist/types/__tests__/llm/client/support/streams.d.ts.map +1 -0
- package/dist/types/__tests__/llm/client/support/transports.d.ts +105 -0
- package/dist/types/__tests__/llm/client/support/transports.d.ts.map +1 -0
- package/dist/types/index.d.ts +1 -0
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/llm/alias-client.d.ts +64 -0
- package/dist/types/llm/alias-client.d.ts.map +1 -0
- package/dist/types/llm/circuit-breaker.d.ts +124 -0
- package/dist/types/llm/circuit-breaker.d.ts.map +1 -0
- package/dist/types/llm/eval/comparators.d.ts +127 -0
- package/dist/types/llm/eval/comparators.d.ts.map +1 -0
- package/dist/types/llm/eval/coverage.d.ts +44 -0
- package/dist/types/llm/eval/coverage.d.ts.map +1 -0
- package/dist/types/llm/eval/golden-set.d.ts +47 -0
- package/dist/types/llm/eval/golden-set.d.ts.map +1 -0
- package/dist/types/llm/eval/index.d.ts +25 -0
- package/dist/types/llm/eval/index.d.ts.map +1 -0
- package/dist/types/llm/eval/json-shape.d.ts +74 -0
- package/dist/types/llm/eval/json-shape.d.ts.map +1 -0
- package/dist/types/llm/eval/judge.d.ts +131 -0
- package/dist/types/llm/eval/judge.d.ts.map +1 -0
- package/dist/types/llm/eval/metrics.d.ts +51 -0
- package/dist/types/llm/eval/metrics.d.ts.map +1 -0
- package/dist/types/llm/eval/run.d.ts +97 -0
- package/dist/types/llm/eval/run.d.ts.map +1 -0
- package/dist/types/llm/eval/types.d.ts +242 -0
- package/dist/types/llm/eval/types.d.ts.map +1 -0
- package/dist/types/llm/fallback-chain.d.ts +97 -0
- package/dist/types/llm/fallback-chain.d.ts.map +1 -0
- package/dist/types/llm/index.d.ts +30 -0
- package/dist/types/llm/index.d.ts.map +1 -0
- package/dist/types/llm/param-matrix.d.ts +65 -0
- package/dist/types/llm/param-matrix.d.ts.map +1 -0
- package/dist/types/llm/rate-guard.d.ts +119 -0
- package/dist/types/llm/rate-guard.d.ts.map +1 -0
- package/dist/types/llm/route-table.d.ts +155 -0
- package/dist/types/llm/route-table.d.ts.map +1 -0
- package/dist/types/llm/schema-retry.d.ts +80 -0
- package/dist/types/llm/schema-retry.d.ts.map +1 -0
- package/dist/types/llm/streaming.d.ts +93 -0
- package/dist/types/llm/streaming.d.ts.map +1 -0
- package/dist/types/llm/transports/direct.d.ts +92 -0
- package/dist/types/llm/transports/direct.d.ts.map +1 -0
- package/dist/types/llm/transports/gateway.d.ts +73 -0
- package/dist/types/llm/transports/gateway.d.ts.map +1 -0
- package/dist/types/llm/types.d.ts +292 -0
- package/dist/types/llm/types.d.ts.map +1 -0
- package/dist/types/schemas/alpaca-schemas.d.ts +6 -6
- package/package.json +2 -2
- package/dist/types/__tests__/alpaca-broker-error-preservation.test.d.ts +0 -2
- package/dist/types/__tests__/alpaca-broker-error-preservation.test.d.ts.map +0 -1
- package/dist/types/__tests__/alpaca-client-order-id.test.d.ts +0 -2
- package/dist/types/__tests__/alpaca-client-order-id.test.d.ts.map +0 -1
- package/dist/types/__tests__/alpaca-functions.test.d.ts +0 -2
- package/dist/types/__tests__/alpaca-functions.test.d.ts.map +0 -1
- package/dist/types/__tests__/alpaca-market-data-retry.test.d.ts +0 -2
- package/dist/types/__tests__/alpaca-market-data-retry.test.d.ts.map +0 -1
- package/dist/types/__tests__/alpaca-order-idempotency.test.d.ts +0 -2
- package/dist/types/__tests__/alpaca-order-idempotency.test.d.ts.map +0 -1
- package/dist/types/__tests__/alpaca-trading-api.test.d.ts +0 -2
- package/dist/types/__tests__/alpaca-trading-api.test.d.ts.map +0 -1
- package/dist/types/__tests__/api-endpoints.test.d.ts +0 -2
- package/dist/types/__tests__/api-endpoints.test.d.ts.map +0 -1
- package/dist/types/__tests__/asset-allocation.test.d.ts +0 -2
- package/dist/types/__tests__/asset-allocation.test.d.ts.map +0 -1
- package/dist/types/__tests__/atr.test.d.ts +0 -2
- package/dist/types/__tests__/atr.test.d.ts.map +0 -1
- package/dist/types/__tests__/auth-validator.test.d.ts +0 -2
- package/dist/types/__tests__/auth-validator.test.d.ts.map +0 -1
- package/dist/types/__tests__/broker-factory.test.d.ts +0 -2
- package/dist/types/__tests__/broker-factory.test.d.ts.map +0 -1
- package/dist/types/__tests__/broker-types.test.d.ts +0 -2
- package/dist/types/__tests__/broker-types.test.d.ts.map +0 -1
- package/dist/types/__tests__/cache.test.d.ts +0 -2
- package/dist/types/__tests__/cache.test.d.ts.map +0 -1
- package/dist/types/__tests__/errors.test.d.ts +0 -2
- package/dist/types/__tests__/errors.test.d.ts.map +0 -1
- package/dist/types/__tests__/financial-regression.test.d.ts +0 -2
- package/dist/types/__tests__/financial-regression.test.d.ts.map +0 -1
- package/dist/types/__tests__/format-tools.test.d.ts +0 -2
- package/dist/types/__tests__/format-tools.test.d.ts.map +0 -1
- package/dist/types/__tests__/http-keep-alive.test.d.ts +0 -2
- package/dist/types/__tests__/http-keep-alive.test.d.ts.map +0 -1
- package/dist/types/__tests__/http-timeout.test.d.ts +0 -2
- package/dist/types/__tests__/http-timeout.test.d.ts.map +0 -1
- package/dist/types/__tests__/index.test.d.ts +0 -2
- package/dist/types/__tests__/index.test.d.ts.map +0 -1
- package/dist/types/__tests__/legacy-auth.test.d.ts +0 -2
- package/dist/types/__tests__/legacy-auth.test.d.ts.map +0 -1
- package/dist/types/__tests__/logger.test.d.ts +0 -2
- package/dist/types/__tests__/logger.test.d.ts.map +0 -1
- package/dist/types/__tests__/logging.test.d.ts +0 -2
- package/dist/types/__tests__/logging.test.d.ts.map +0 -1
- package/dist/types/__tests__/market-time.test.d.ts +0 -2
- package/dist/types/__tests__/market-time.test.d.ts.map +0 -1
- package/dist/types/__tests__/massive.test.d.ts +0 -2
- package/dist/types/__tests__/massive.test.d.ts.map +0 -1
- package/dist/types/__tests__/metrics-calcs-direction.test.d.ts +0 -2
- package/dist/types/__tests__/metrics-calcs-direction.test.d.ts.map +0 -1
- package/dist/types/__tests__/misc-utils.test.d.ts +0 -2
- package/dist/types/__tests__/misc-utils.test.d.ts.map +0 -1
- package/dist/types/__tests__/paginator.test.d.ts +0 -2
- package/dist/types/__tests__/paginator.test.d.ts.map +0 -1
- package/dist/types/__tests__/performance-metrics-fees.test.d.ts +0 -2
- package/dist/types/__tests__/performance-metrics-fees.test.d.ts.map +0 -1
- package/dist/types/__tests__/performance-metrics.test.d.ts +0 -2
- package/dist/types/__tests__/performance-metrics.test.d.ts.map +0 -1
- package/dist/types/__tests__/price-utils-fees.test.d.ts +0 -2
- package/dist/types/__tests__/price-utils-fees.test.d.ts.map +0 -1
- package/dist/types/__tests__/price-utils.test.d.ts +0 -2
- package/dist/types/__tests__/price-utils.test.d.ts.map +0 -1
- package/dist/types/__tests__/property-based-financial.test.d.ts +0 -2
- package/dist/types/__tests__/property-based-financial.test.d.ts.map +0 -1
- package/dist/types/__tests__/protective-order-sides.test.d.ts +0 -2
- package/dist/types/__tests__/protective-order-sides.test.d.ts.map +0 -1
- package/dist/types/__tests__/rate-limiter.test.d.ts +0 -2
- package/dist/types/__tests__/rate-limiter.test.d.ts.map +0 -1
- package/dist/types/__tests__/retry-classification.test.d.ts +0 -2
- package/dist/types/__tests__/retry-classification.test.d.ts.map +0 -1
- package/dist/types/__tests__/retry.test.d.ts +0 -2
- package/dist/types/__tests__/retry.test.d.ts.map +0 -1
- package/dist/types/__tests__/risk-free-rate.test.d.ts +0 -2
- package/dist/types/__tests__/risk-free-rate.test.d.ts.map +0 -1
- package/dist/types/__tests__/risk-metrics.test.d.ts +0 -2
- package/dist/types/__tests__/risk-metrics.test.d.ts.map +0 -1
- package/dist/types/__tests__/schema-validation.test.d.ts +0 -2
- package/dist/types/__tests__/schema-validation.test.d.ts.map +0 -1
- package/dist/types/__tests__/stampede-load-timeout.test.d.ts +0 -2
- package/dist/types/__tests__/stampede-load-timeout.test.d.ts.map +0 -1
- package/dist/types/__tests__/strategy-metrics.test.d.ts +0 -2
- package/dist/types/__tests__/strategy-metrics.test.d.ts.map +0 -1
- package/dist/types/__tests__/technical-analysis-totality.test.d.ts +0 -2
- package/dist/types/__tests__/technical-analysis-totality.test.d.ts.map +0 -1
- package/dist/types/__tests__/technical-analysis.test.d.ts +0 -2
- package/dist/types/__tests__/technical-analysis.test.d.ts.map +0 -1
- package/dist/types/__tests__/time-utils.test.d.ts +0 -2
- package/dist/types/__tests__/time-utils.test.d.ts.map +0 -1
- package/dist/types/__tests__/trading-policy-schemas.test.d.ts +0 -2
- package/dist/types/__tests__/trading-policy-schemas.test.d.ts.map +0 -1
- package/dist/types/__tests__/trailing-stops-portfolio.test.d.ts +0 -2
- package/dist/types/__tests__/trailing-stops-portfolio.test.d.ts.map +0 -1
- package/dist/types/__tests__/volatility.test.d.ts +0 -2
- package/dist/types/__tests__/volatility.test.d.ts.map +0 -1
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Typed access to the error a call was supposed to raise.
|
|
3
|
+
*
|
|
4
|
+
* `await promise.catch((error) => error)` types the result as the union of the
|
|
5
|
+
* error and the value the promise might have resolved to, which erases exactly
|
|
6
|
+
* the properties an assertion wants to read. Worse, a call that wrongly
|
|
7
|
+
* SUCCEEDS then surfaces as a missing property rather than as the contract
|
|
8
|
+
* violation it is. This narrows once, and fails loudly on a resolution.
|
|
9
|
+
*
|
|
10
|
+
* @module __tests__/llm/client/support/rejections
|
|
11
|
+
*/
|
|
12
|
+
/**
|
|
13
|
+
* Await a call that must fail, and return its error at the expected type.
|
|
14
|
+
*
|
|
15
|
+
* @param promise The call under test.
|
|
16
|
+
* @param expected The error class the call is contracted to raise.
|
|
17
|
+
* @returns The raised error.
|
|
18
|
+
* @throws When the call resolves, or raises something of another type.
|
|
19
|
+
*/
|
|
20
|
+
export declare function rejection<E extends Error>(promise: Promise<unknown>, expected: new (...args: never[]) => E): Promise<E>;
|
|
21
|
+
//# sourceMappingURL=rejections.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rejections.d.ts","sourceRoot":"","sources":["../../../../../../src/__tests__/llm/client/support/rejections.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH;;;;;;;GAOG;AACH,wBAAsB,SAAS,CAAC,CAAC,SAAS,KAAK,EAC7C,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,EACzB,QAAQ,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,GACpC,OAAO,CAAC,CAAC,CAAC,CAUZ"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Route fixtures for the alias-client test suite.
|
|
3
|
+
*
|
|
4
|
+
* The canonical route table is the product's routing policy, not a test
|
|
5
|
+
* fixture: most of its open-weight legs are deliberately unconfirmed today, and
|
|
6
|
+
* no leg declares a missing capability. Exercising a three-leg walk, or a
|
|
7
|
+
* provider that rejects a sampling parameter, therefore needs routes built
|
|
8
|
+
* here. They are built from the same public `ResolvedRoute` shape the resolver
|
|
9
|
+
* emits, so a change to that shape breaks these fixtures rather than letting
|
|
10
|
+
* them drift into describing a chain the client can no longer produce.
|
|
11
|
+
*
|
|
12
|
+
* @module __tests__/llm/client/support/routes
|
|
13
|
+
*/
|
|
14
|
+
import type { LlmAlias, LlmProvider, LlmRouteParams, LlmRouteRole, ResolvedRoute } from "../../../../llm/types";
|
|
15
|
+
/** Per-leg budget used wherever a test needs a timeout it can drive by hand. */
|
|
16
|
+
export declare const TEST_LEG_TIMEOUT_MS = 1000;
|
|
17
|
+
/** Retry budget carried on fixture routes; the chain itself does the retrying. */
|
|
18
|
+
export declare const TEST_RETRIES_PER_LEG = 1;
|
|
19
|
+
/** An open-weight host: an OpenAI-compatible wire format the degraded path must refuse. */
|
|
20
|
+
export declare const OPEN_PROVIDER: LlmProvider;
|
|
21
|
+
/** A retained closed vendor speaking the Anthropic wire format (PD-11). */
|
|
22
|
+
export declare const CLOSED_PROVIDER: LlmProvider;
|
|
23
|
+
/** An aggregator backstop, used where a third tier value has to be distinguishable. */
|
|
24
|
+
export declare const AGGREGATOR_PROVIDER: LlmProvider;
|
|
25
|
+
/** Fields a fixture route may override; everything else takes a serving default. */
|
|
26
|
+
export interface RouteOverrides {
|
|
27
|
+
readonly alias?: LlmAlias;
|
|
28
|
+
readonly isolated?: boolean;
|
|
29
|
+
readonly role?: LlmRouteRole;
|
|
30
|
+
readonly provider?: LlmProvider;
|
|
31
|
+
readonly providerName?: string;
|
|
32
|
+
readonly modelId?: string;
|
|
33
|
+
readonly lumicModel?: string | null;
|
|
34
|
+
readonly params?: LlmRouteParams;
|
|
35
|
+
readonly timeoutMs?: number;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Build one resolved leg.
|
|
39
|
+
*
|
|
40
|
+
* The route key is derived through the production `routeKeyFor` rather than
|
|
41
|
+
* spelled out, so a fixture and the breaker registry can never disagree about
|
|
42
|
+
* which leg a key names.
|
|
43
|
+
*
|
|
44
|
+
* @param overrides Fields to set; the rest take serving defaults.
|
|
45
|
+
* @returns A resolved leg ready to hand to the chain executor.
|
|
46
|
+
*/
|
|
47
|
+
export declare function makeRoute(overrides?: RouteOverrides): ResolvedRoute;
|
|
48
|
+
/**
|
|
49
|
+
* Build a full primary -> secondary -> closed-incumbent chain.
|
|
50
|
+
*
|
|
51
|
+
* @param alias The alias the legs belong to.
|
|
52
|
+
* @returns The three legs in chain order.
|
|
53
|
+
*/
|
|
54
|
+
export declare function makeThreeLegChain(alias?: LlmAlias): ResolvedRoute[];
|
|
55
|
+
/**
|
|
56
|
+
* Name an alias the type union deliberately forbids.
|
|
57
|
+
*
|
|
58
|
+
* `LlmAlias` is a closed union precisely so a mistyped alias cannot compile,
|
|
59
|
+
* which leaves the resolver's runtime guard unreachable from well-typed code.
|
|
60
|
+
* Reaching it requires one deliberate widening; it is confined here so the
|
|
61
|
+
* assertion appears once rather than at each call site that tests the guard.
|
|
62
|
+
*
|
|
63
|
+
* @param name The alias name to pass through.
|
|
64
|
+
* @returns The same name, typed as an alias.
|
|
65
|
+
*/
|
|
66
|
+
export declare function asAlias(name: string): LlmAlias;
|
|
67
|
+
//# sourceMappingURL=routes.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"routes.d.ts","sourceRoot":"","sources":["../../../../../../src/__tests__/llm/client/support/routes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,KAAK,EACV,QAAQ,EACR,WAAW,EACX,cAAc,EACd,YAAY,EACZ,aAAa,EACd,MAAM,uBAAuB,CAAC;AAE/B,gFAAgF;AAChF,eAAO,MAAM,mBAAmB,OAAQ,CAAC;AAEzC,kFAAkF;AAClF,eAAO,MAAM,oBAAoB,IAAI,CAAC;AAEtC,2FAA2F;AAC3F,eAAO,MAAM,aAAa,EAAE,WAU3B,CAAC;AAEF,2EAA2E;AAC3E,eAAO,MAAM,eAAe,EAAE,WAW7B,CAAC;AAEF,uFAAuF;AACvF,eAAO,MAAM,mBAAmB,EAAE,WAUjC,CAAC;AAEF,oFAAoF;AACpF,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC;IAC1B,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC;IAChC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,MAAM,CAAC,EAAE,cAAc,CAAC;IACjC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;;;;;;;;;GASG;AACH,wBAAgB,SAAS,CAAC,SAAS,GAAE,cAAmB,GAAG,aAAa,CAkBvE;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,GAAE,QAAwB,GAAG,aAAa,EAAE,CAgBlF;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,CAE9C"}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Wire-format stream builders for the streaming normalisation tests.
|
|
3
|
+
*
|
|
4
|
+
* The normaliser's hardest requirement is that a chunk boundary is not a
|
|
5
|
+
* message boundary: the network splits bytes wherever it likes, including
|
|
6
|
+
* inside a JSON payload. These builders therefore emit the wire text and let a
|
|
7
|
+
* test choose the byte-chunking independently, so the same logical stream can
|
|
8
|
+
* be replayed whole, split per line, or split every few bytes.
|
|
9
|
+
*
|
|
10
|
+
* @module __tests__/llm/client/support/streams
|
|
11
|
+
*/
|
|
12
|
+
/** Sentinel closing an OpenAI-compatible server-sent event stream. */
|
|
13
|
+
export declare const SSE_DONE_LINE = "data: [DONE]";
|
|
14
|
+
/**
|
|
15
|
+
* Render an OpenAI-compatible SSE body.
|
|
16
|
+
*
|
|
17
|
+
* @param deltas Text increments, in order.
|
|
18
|
+
* @param options Rendering options.
|
|
19
|
+
* @param options.terminate Emit the `[DONE]` sentinel. Omitting it models a truncated stream.
|
|
20
|
+
* @param options.errorAfter Emit a provider error event after this many deltas.
|
|
21
|
+
* @returns The wire text.
|
|
22
|
+
*/
|
|
23
|
+
export declare function openAiSseBody(deltas: readonly string[], options?: {
|
|
24
|
+
readonly terminate?: boolean;
|
|
25
|
+
readonly errorAfter?: number;
|
|
26
|
+
}): string;
|
|
27
|
+
/**
|
|
28
|
+
* Render an Anthropic event-stream body.
|
|
29
|
+
*
|
|
30
|
+
* Each event carries both the `event:` line a real stream sends and the `data:`
|
|
31
|
+
* line the normaliser reads, so the test proves the normaliser ignores the
|
|
32
|
+
* former rather than merely never meeting it.
|
|
33
|
+
*
|
|
34
|
+
* @param deltas Text increments, in order.
|
|
35
|
+
* @param options Rendering options.
|
|
36
|
+
* @param options.terminate Emit `message_stop`. Omitting it models a truncated stream.
|
|
37
|
+
* @param options.errorAfter Emit a provider error event after this many deltas.
|
|
38
|
+
* @returns The wire text.
|
|
39
|
+
*/
|
|
40
|
+
export declare function anthropicSseBody(deltas: readonly string[], options?: {
|
|
41
|
+
readonly terminate?: boolean;
|
|
42
|
+
readonly errorAfter?: number;
|
|
43
|
+
}): string;
|
|
44
|
+
/**
|
|
45
|
+
* Split text into fixed-size pieces.
|
|
46
|
+
*
|
|
47
|
+
* Sized in characters rather than lines so a piece can land inside a JSON
|
|
48
|
+
* payload, which is the boundary case a line-oriented parser gets wrong.
|
|
49
|
+
*
|
|
50
|
+
* @param text The wire text.
|
|
51
|
+
* @param size Characters per piece.
|
|
52
|
+
* @returns The pieces, in order.
|
|
53
|
+
*/
|
|
54
|
+
export declare function splitEvery(text: string, size: number): string[];
|
|
55
|
+
/**
|
|
56
|
+
* Present pieces of wire text as the byte stream a response body would be.
|
|
57
|
+
*
|
|
58
|
+
* @param pieces The pieces, in order.
|
|
59
|
+
* @returns An async iterable of encoded chunks.
|
|
60
|
+
*/
|
|
61
|
+
export declare function byteStream(pieces: readonly string[]): AsyncIterable<Uint8Array>;
|
|
62
|
+
/**
|
|
63
|
+
* Present wire text as a byte stream chopped at fixed BYTE offsets.
|
|
64
|
+
*
|
|
65
|
+
* Chopping the encoded bytes rather than the string is what puts a boundary
|
|
66
|
+
* inside a multi-byte character as well as inside a JSON payload, which is the
|
|
67
|
+
* case an incremental decoder exists for.
|
|
68
|
+
*
|
|
69
|
+
* @param text The wire text.
|
|
70
|
+
* @param size Bytes per chunk.
|
|
71
|
+
* @returns An async iterable of encoded chunks.
|
|
72
|
+
*/
|
|
73
|
+
export declare function byteStreamSplitEvery(text: string, size: number): AsyncIterable<Uint8Array>;
|
|
74
|
+
//# sourceMappingURL=streams.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"streams.d.ts","sourceRoot":"","sources":["../../../../../../src/__tests__/llm/client/support/streams.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,sEAAsE;AACtE,eAAO,MAAM,aAAa,iBAAiB,CAAC;AAE5C;;;;;;;;GAQG;AACH,wBAAgB,aAAa,CAC3B,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,OAAO,GAAE;IAAE,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAO,GAC3E,MAAM,CAqBR;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,OAAO,GAAE;IAAE,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAO,GAC3E,MAAM,CA4BR;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAM/D;AAED;;;;;GAKG;AACH,wBAAgB,UAAU,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,aAAa,CAAC,UAAU,CAAC,CAS/E;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,GACX,aAAa,CAAC,UAAU,CAAC,CAS3B"}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scripted transport doubles for the alias-client test suite.
|
|
3
|
+
*
|
|
4
|
+
* The client's guarantees are all about what it does BETWEEN transports —
|
|
5
|
+
* which leg it tries next, how long it waits, what it records, what it refuses
|
|
6
|
+
* to invent. Observing those needs a transport whose every outcome is chosen by
|
|
7
|
+
* the test and whose every invocation is recorded, so an assertion can be made
|
|
8
|
+
* about a leg that was never called at all. A real provider, or a network
|
|
9
|
+
* mock, would make exactly those assertions unavailable.
|
|
10
|
+
*
|
|
11
|
+
* @module __tests__/llm/client/support/transports
|
|
12
|
+
*/
|
|
13
|
+
import type { LlmResponseFormat, LlmToolCall, LlmTransport, LlmTransportRequest, LlmTransportResponse, LlmUsageRecord, ResolvedRoute } from "../../../../llm/types";
|
|
14
|
+
/** One invocation, captured for assertions about routing and normalisation. */
|
|
15
|
+
export interface ScriptedCall {
|
|
16
|
+
/** Zero-based invocation order across the whole transport. */
|
|
17
|
+
readonly index: number;
|
|
18
|
+
readonly route: ResolvedRoute;
|
|
19
|
+
readonly content: string | readonly unknown[];
|
|
20
|
+
readonly params: Readonly<Record<string, unknown>>;
|
|
21
|
+
readonly responseFormat: LlmResponseFormat;
|
|
22
|
+
/** The per-leg signal, kept live so a test can observe an abort after the fact. */
|
|
23
|
+
readonly signal: AbortSignal;
|
|
24
|
+
readonly correlationId?: string;
|
|
25
|
+
}
|
|
26
|
+
/** What a scripted leg does when invoked. */
|
|
27
|
+
export type LegBehaviour = {
|
|
28
|
+
readonly kind: "ok";
|
|
29
|
+
readonly response: unknown;
|
|
30
|
+
readonly usage: LlmUsageRecord;
|
|
31
|
+
readonly toolCalls?: readonly LlmToolCall[];
|
|
32
|
+
} | {
|
|
33
|
+
readonly kind: "fail";
|
|
34
|
+
readonly error: Error;
|
|
35
|
+
}
|
|
36
|
+
/** Never settles on its own: the leg ends only when its signal aborts. */
|
|
37
|
+
| {
|
|
38
|
+
readonly kind: "hang";
|
|
39
|
+
};
|
|
40
|
+
/** Token counts a scripted leg reports. */
|
|
41
|
+
export interface UsageOverrides {
|
|
42
|
+
readonly promptTokens?: number;
|
|
43
|
+
readonly completionTokens?: number;
|
|
44
|
+
readonly cost?: number;
|
|
45
|
+
readonly provider?: string;
|
|
46
|
+
readonly model?: string;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Build a usage record attributed to a leg.
|
|
50
|
+
*
|
|
51
|
+
* @param route The leg the usage belongs to.
|
|
52
|
+
* @param overrides Counts to report.
|
|
53
|
+
* @returns The usage record.
|
|
54
|
+
*/
|
|
55
|
+
export declare function usageFor(route: ResolvedRoute, overrides?: UsageOverrides): LlmUsageRecord;
|
|
56
|
+
/**
|
|
57
|
+
* A leg that answers.
|
|
58
|
+
*
|
|
59
|
+
* @param response The payload to return.
|
|
60
|
+
* @param usage The usage to report.
|
|
61
|
+
* @param toolCalls Tool calls the model asked for, if any.
|
|
62
|
+
* @returns The behaviour.
|
|
63
|
+
*/
|
|
64
|
+
export declare function answers(response: unknown, usage: LlmUsageRecord, toolCalls?: readonly LlmToolCall[]): LegBehaviour;
|
|
65
|
+
/**
|
|
66
|
+
* A leg that throws.
|
|
67
|
+
*
|
|
68
|
+
* @param error The error to throw.
|
|
69
|
+
* @returns The behaviour.
|
|
70
|
+
*/
|
|
71
|
+
export declare function fails(error: Error): LegBehaviour;
|
|
72
|
+
/**
|
|
73
|
+
* A leg that never answers until its signal aborts.
|
|
74
|
+
*
|
|
75
|
+
* @returns The behaviour.
|
|
76
|
+
*/
|
|
77
|
+
export declare function hangs(): LegBehaviour;
|
|
78
|
+
/**
|
|
79
|
+
* A transport whose every outcome is chosen by the test and whose every
|
|
80
|
+
* invocation is recorded.
|
|
81
|
+
*/
|
|
82
|
+
export declare class ScriptedTransport implements LlmTransport {
|
|
83
|
+
/** Identifies which transport answered, so degraded-path tests can tell them apart. */
|
|
84
|
+
readonly name: string;
|
|
85
|
+
/** Every invocation, in order. */
|
|
86
|
+
readonly calls: ScriptedCall[];
|
|
87
|
+
private readonly plan;
|
|
88
|
+
/**
|
|
89
|
+
* @param name Transport name, surfaced on the `LlmTransport` contract.
|
|
90
|
+
* @param plan Decides what each invocation does, given the recorded call.
|
|
91
|
+
*/
|
|
92
|
+
constructor(name: string, plan: (call: ScriptedCall) => LegBehaviour);
|
|
93
|
+
/** Route keys invoked, in order. */
|
|
94
|
+
get routeKeys(): string[];
|
|
95
|
+
/** Provider tiers invoked, in order, for the degraded path's closed-only rule. */
|
|
96
|
+
get tiers(): string[];
|
|
97
|
+
/**
|
|
98
|
+
* Execute one scripted leg.
|
|
99
|
+
*
|
|
100
|
+
* @param request The normalised leg request.
|
|
101
|
+
* @returns Whatever the plan decided for this invocation.
|
|
102
|
+
*/
|
|
103
|
+
execute<T>(request: LlmTransportRequest): Promise<LlmTransportResponse<T>>;
|
|
104
|
+
}
|
|
105
|
+
//# sourceMappingURL=transports.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transports.d.ts","sourceRoot":"","sources":["../../../../../../src/__tests__/llm/client/support/transports.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EACV,iBAAiB,EACjB,WAAW,EACX,YAAY,EACZ,mBAAmB,EACnB,oBAAoB,EACpB,cAAc,EACd,aAAa,EACd,MAAM,uBAAuB,CAAC;AAE/B,+EAA+E;AAC/E,MAAM,WAAW,YAAY;IAC3B,8DAA8D;IAC9D,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,OAAO,EAAE,CAAC;IAC9C,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IACnD,QAAQ,CAAC,cAAc,EAAE,iBAAiB,CAAC;IAC3C,mFAAmF;IACnF,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;CACjC;AAED,6CAA6C;AAC7C,MAAM,MAAM,YAAY,GACpB;IACE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IACpB,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,cAAc,CAAC;IAC/B,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,WAAW,EAAE,CAAC;CAC7C,GACD;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAA;CAAE;AAClD,0EAA0E;GACxE;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAE9B,2CAA2C;AAC3C,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;GAMG;AACH,wBAAgB,QAAQ,CACtB,KAAK,EAAE,aAAa,EACpB,SAAS,GAAE,cAAmB,GAC7B,cAAc,CAQhB;AAED;;;;;;;GAOG;AACH,wBAAgB,OAAO,CACrB,QAAQ,EAAE,OAAO,EACjB,KAAK,EAAE,cAAc,EACrB,SAAS,CAAC,EAAE,SAAS,WAAW,EAAE,GACjC,YAAY,CAEd;AAED;;;;;GAKG;AACH,wBAAgB,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,YAAY,CAEhD;AAED;;;;GAIG;AACH,wBAAgB,KAAK,IAAI,YAAY,CAEpC;AAkBD;;;GAGG;AACH,qBAAa,iBAAkB,YAAW,YAAY;IACpD,uFAAuF;IACvF,SAAgB,IAAI,EAAE,MAAM,CAAC;IAE7B,kCAAkC;IAClC,SAAgB,KAAK,EAAE,YAAY,EAAE,CAAM;IAE3C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAuC;IAE5D;;;OAGG;gBACgB,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,YAAY,KAAK,YAAY;IAK3E,oCAAoC;IACpC,IAAW,SAAS,IAAI,MAAM,EAAE,CAE/B;IAED,kFAAkF;IAClF,IAAW,KAAK,IAAI,MAAM,EAAE,CAE3B;IAED;;;;;OAKG;IACU,OAAO,CAAC,CAAC,EACpB,OAAO,EAAE,mBAAmB,GAC3B,OAAO,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC;CAwCpC"}
|
package/dist/types/index.d.ts
CHANGED
|
@@ -19,6 +19,7 @@ import * as ta from "./technical-analysis";
|
|
|
19
19
|
import * as tu from "./time-utils";
|
|
20
20
|
import * as Types from "./types";
|
|
21
21
|
import * as volatilityNs from "./volatility";
|
|
22
|
+
export * from "./llm";
|
|
22
23
|
export { getLogger, resetLogger, setLogger, type Logger } from "./logger";
|
|
23
24
|
export { AdapticUtilsError, type AlpacaBrokerErrorDetail, AlpacaApiError, AlphaVantageError, AuthenticationError, DataFormatError, DuplicateClientOrderIdError, enrichAlpacaError, extractAlpacaBrokerError, getAlpacaBrokerErrorCode, getAlpacaBrokerErrorDetail, HttpClientError, HttpServerError, MassiveApiError, NetworkError, RateLimitError, TimeoutError, UnsupportedBrokerError, ValidationError, WebSocketError, } from "./errors";
|
|
24
25
|
export { validateAlpacaCredentials, validateAlphaVantageApiKey, validateMassiveApiKey, } from "./utils/auth-validator";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,KAAK,MAAM,MAAM,iBAAiB,CAAC;AAC1C,OAAO,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACrC,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAC/B,OAAO,KAAK,MAAM,MAAM,UAAU,CAAC;AACnC,OAAO,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACrC,OAAO,KAAK,EAAE,MAAM,eAAe,CAAC;AACpC,OAAO,KAAK,OAAO,MAAM,WAAW,CAAC;AAErC,OAAO,iBAAiB,MAAM,iBAAiB,CAAC;AAChD,OAAO,KAAK,IAAI,MAAM,cAAc,CAAC;AACrC,OAAO,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,eAAe,CAAC;AACpC,OAAO,EAAE,sBAAsB,EAAgB,MAAM,gBAAgB,CAAC;AACtE,OAAO,KAAK,MAAM,MAAM,gBAAgB,CAAC;AACzC,OAAO,KAAK,UAAU,MAAM,oBAAoB,CAAC;AACjD,OAAO,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC3C,OAAO,KAAK,EAAE,MAAM,cAAc,CAAC;AACnC,OAAO,KAAK,KAAK,MAAM,SAAS,CAAC;AACjC,OAAO,KAAK,YAAY,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,KAAK,MAAM,MAAM,iBAAiB,CAAC;AAC1C,OAAO,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACrC,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAC/B,OAAO,KAAK,MAAM,MAAM,UAAU,CAAC;AACnC,OAAO,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACrC,OAAO,KAAK,EAAE,MAAM,eAAe,CAAC;AACpC,OAAO,KAAK,OAAO,MAAM,WAAW,CAAC;AAErC,OAAO,iBAAiB,MAAM,iBAAiB,CAAC;AAChD,OAAO,KAAK,IAAI,MAAM,cAAc,CAAC;AACrC,OAAO,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,eAAe,CAAC;AACpC,OAAO,EAAE,sBAAsB,EAAgB,MAAM,gBAAgB,CAAC;AACtE,OAAO,KAAK,MAAM,MAAM,gBAAgB,CAAC;AACzC,OAAO,KAAK,UAAU,MAAM,oBAAoB,CAAC;AACjD,OAAO,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC3C,OAAO,KAAK,EAAE,MAAM,cAAc,CAAC;AACnC,OAAO,KAAK,KAAK,MAAM,SAAS,CAAC;AACjC,OAAO,KAAK,YAAY,MAAM,cAAc,CAAC;AAc7C,cAAc,OAAO,CAAC;AAEtB,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,SAAS,EAAE,KAAK,MAAM,EAAE,MAAM,UAAU,CAAC;AAG1E,OAAO,EACL,iBAAiB,EACjB,KAAK,uBAAuB,EAC5B,cAAc,EACd,iBAAiB,EACjB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,iBAAiB,EACjB,wBAAwB,EACxB,wBAAwB,EACxB,0BAA0B,EAC1B,eAAe,EACf,eAAe,EACf,eAAe,EACf,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,sBAAsB,EACtB,eAAe,EACf,cAAc,GACf,MAAM,UAAU,CAAC;AAGlB,OAAO,EACL,yBAAyB,EACzB,0BAA0B,EAC1B,qBAAqB,GACtB,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EACL,eAAe,EACf,WAAW,EACX,iBAAiB,EACjB,kBAAkB,EAClB,mBAAmB,EACnB,iBAAiB,EACjB,gBAAgB,EAChB,sBAAsB,EACtB,KAAK,WAAW,GACjB,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EACL,qBAAqB,EACrB,sBAAsB,EACtB,4BAA4B,EAC5B,KAAK,UAAU,EACf,KAAK,cAAc,EACnB,KAAK,WAAW,EAChB,KAAK,UAAU,EACf,KAAK,6BAA6B,GACnC,MAAM,kCAAkC,CAAC;AAG1C,OAAO,EACL,sBAAsB,EACtB,YAAY,EACZ,KAAK,iBAAiB,GACvB,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EACL,iBAAiB,EACjB,uBAAuB,EACvB,SAAS,EACT,KAAK,WAAW,GACjB,MAAM,eAAe,CAAC;AAIvB,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAGhD,OAAO,EACL,gBAAgB,EAChB,mBAAmB,EACnB,UAAU,EACV,WAAW,GACZ,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EACL,qBAAqB,EACrB,yBAAyB,EACzB,qBAAqB,GACtB,MAAM,8BAA8B,CAAC;AAEtC,cAAc,gCAAgC,CAAC;AAG/C,OAAO,EACL,mBAAmB,EACnB,oBAAoB,EAEpB,0BAA0B,EAC1B,eAAe,EACf,8BAA8B,EAC9B,kCAAkC,EAClC,8BAA8B,EAC9B,gCAAgC,EAChC,gCAAgC,EAChC,uBAAuB,EACvB,wBAAwB,EACxB,iBAAiB,EACjB,uBAAuB,EACvB,oCAAoC,EACpC,oBAAoB,EACpB,0BAA0B,EAC1B,iBAAiB,EACjB,iBAAiB,EAEjB,+BAA+B,EAC/B,+BAA+B,EAC/B,2BAA2B,EAC3B,0BAA0B,EAC1B,iCAAiC,EACjC,8BAA8B,EAC9B,kCAAkC,EAClC,uBAAuB,EACvB,kBAAkB,IAAI,qBAAqB,EAC3C,2BAA2B,EAE3B,yBAAyB,EACzB,uBAAuB,EACvB,oBAAoB,EACpB,gBAAgB,EAChB,KAAK,uBAAuB,EAC5B,KAAK,gBAAgB,GACtB,MAAM,WAAW,CAAC;AAGnB,OAAO,EACL,QAAQ,EACR,WAAW,EACX,KAAK,sBAAsB,EAC3B,KAAK,sBAAsB,EAC3B,KAAK,gBAAgB,EACrB,KAAK,mBAAmB,GACzB,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EACL,mBAAmB,EACnB,kBAAkB,EAClB,SAAS,EACT,UAAU,EACV,oBAAoB,EACpB,KAAK,oBAAoB,GAC1B,MAAM,yBAAyB,CAAC;AAGjC,OAAO,EACL,sBAAsB,EACtB,qBAAqB,EACrB,eAAe,EACf,6BAA6B,EAC7B,yBAAyB,EACzB,uCAAuC,EACvC,eAAe,EACf,sBAAsB,EACtB,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,GACxB,MAAM,kBAAkB,CAAC;AAG1B,cAAc,SAAS,CAAC;AAGxB,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAGxD,eAAO,MAAM,sBAAsB,GACjC,aAAa,KAAK,CAAC,iBAAiB,qBAGrC,CAAC;AAEF,eAAO,MAAM,yBAAyB,2BAErC,CAAC;AAGF,cAAc,UAAU,CAAC;AAKzB,OAAO,EACL,kBAAkB,EAClB,KAAK,kBAAkB,EACvB,KAAK,mBAAmB,EACxB,KAAK,0BAA0B,GAChC,MAAM,UAAU,CAAC;AAGlB,OAAO,KAAK,aAAa,MAAM,kBAAkB,CAAC;AAGlD,OAAO,KAAK,GAAG,MAAM,OAAO,CAAC;AAC7B,OAAO,KAAK,UAAU,MAAM,cAAc,CAAC;AAC3C,OAAO,KAAK,IAAI,MAAM,gBAAgB,CAAC;AACvC,OAAO,KAAK,QAAQ,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EACL,YAAY,EACZ,WAAW,EACX,eAAe,EACf,aAAa,EACb,eAAe,EACf,oBAAoB,EACpB,qBAAqB,EACrB,WAAW,GACZ,MAAM,wBAAwB,CAAC;AAChC,YAAY,EACV,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,sBAAsB,EACtB,cAAc,EACd,uBAAuB,EACvB,0BAA0B,EAC1B,oBAAoB,EACpB,UAAU,EACV,sBAAsB,EACtB,cAAc,EACd,sBAAsB,GACvB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,sBAAsB,EAAE,MAAM,kDAAkD,CAAC;AAG1F,YAAY,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE/C,eAAO,MAAM,OAAO;;;;;;;;;;;;;;;;;QAoBhB,oEAAoE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oEA2K1D,CAAC;;;;;;;;;;;;;;;;;4BA5KkB,CAAC;0BACnB,CAAC;;;;;;;;;;;QAEZ,iFAAiF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAEjF,2EAA2E;;QAE3E,6FAA6F;;QAE7F,yFAAyF;;QAIzF,4EAA4E;;;;;;;;;;QAU5E,sEAAsE;;;;;;;;QAQtE,sEAAsE;;QAEtE,mDAAmD;;QAEnD,iDAAiD;;QAEjD,8BAA8B;;QAE9B,+BAA+B;;QAE/B,4CAA4C;;;;;;;QAE5C,mDAAmD;;;;;QAEnD,+CAA+C;;QAI/C,+CAA+C;;;;;;;;QAQ/C,wCAAwC;;QAExC,+DAA+D;;QAE/D,mEAAmE;;QAEnE,sEAAsE;;QAEtE,kDAAkD;;;;QAIlD,4CAA4C;;;;;;;kBAvUlC,CAAC;;;iBA4HQ,CAAC;eACtB,CAAA;iBAAiB,CAAC;kBACP,CAAC;gBAAkB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAkBT,CAAC;;;kBAkTshU,CAAC;oBAAsB,CAAC;sBAAyB,CAAC;;;kBAvL3kU,CAAC;;;kBA2GnB,CAAH;;;kBA4EspiB,CAAC;qBAAuB,CAAC;uBAAkC,CAAC;wBAAmC,CAAC;uBAAkC,CAAC;wBAAmC,CAAC;iBAA4B,CAAC;iBAA2B,CAAC;gBAAkB,CAAC;;;;;eAAp/gB,CAAC;;;iBAAiE,CAAC;oBAAsB,CAAC;;kBAAoC,CAAC;;;;;eAA48L,CAAC;;;iBAAiE,CAAC;oBAAsB,CAAC;;kBAAoC,CAAC;;;;;kBAAk0N,CAAC;oBAAsB,CAAC;;;;;;kBAza96b,CAAC;;;kBA+EJ,CAAC;;;kBA0DR,CAAA;;;kBA+C4C,CAAC;;;kBA0DiC,CAAC;gBAC5E,CAAC;iBAAmB,CAAC;iBAEZ,CAAA;gBAAmB,CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAiK7B;;;WAGG;;QAEH;;;;WAIG;;;;;;;;;;;CAON,CAAC;AAEF,eAAO,MAAM,KAAK;;;;;;;;;;;;;;;;;QA9Ld,oEAAoE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oEA2K1D,CAAC;;;;;;;;;;;;;;;;;4BA5KkB,CAAC;0BACnB,CAAC;;;;;;;;;;;QAEZ,iFAAiF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAEjF,2EAA2E;;QAE3E,6FAA6F;;QAE7F,yFAAyF;;QAIzF,4EAA4E;;;;;;;;;;QAU5E,sEAAsE;;;;;;;;QAQtE,sEAAsE;;QAEtE,mDAAmD;;QAEnD,iDAAiD;;QAEjD,8BAA8B;;QAE9B,+BAA+B;;QAE/B,4CAA4C;;;;;;;QAE5C,mDAAmD;;;;;QAEnD,+CAA+C;;QAI/C,+CAA+C;;;;;;;;QAQ/C,wCAAwC;;QAExC,+DAA+D;;QAE/D,mEAAmE;;QAEnE,sEAAsE;;QAEtE,kDAAkD;;;;QAIlD,4CAA4C;;;;;;;kBAvUlC,CAAC;;;iBA4HQ,CAAC;eACtB,CAAA;iBAAiB,CAAC;kBACP,CAAC;gBAAkB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAkBT,CAAC;;;kBAkTshU,CAAC;oBAAsB,CAAC;sBAAyB,CAAC;;;kBAvL3kU,CAAC;;;kBA2GnB,CAAH;;;kBA4EspiB,CAAC;qBAAuB,CAAC;uBAAkC,CAAC;wBAAmC,CAAC;uBAAkC,CAAC;wBAAmC,CAAC;iBAA4B,CAAC;iBAA2B,CAAC;gBAAkB,CAAC;;;;;eAAp/gB,CAAC;;;iBAAiE,CAAC;oBAAsB,CAAC;;kBAAoC,CAAC;;;;;eAA48L,CAAC;;;iBAAiE,CAAC;oBAAsB,CAAC;;kBAAoC,CAAC;;;;;kBAAk0N,CAAC;oBAAsB,CAAC;;;;;;kBAza96b,CAAC;;;kBA+EJ,CAAC;;;kBA0DR,CAAA;;;kBA+C4C,CAAC;;;kBA0DiC,CAAC;gBAC5E,CAAC;iBAAmB,CAAC;iBAEZ,CAAA;gBAAmB,CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAiK7B;;;WAGG;;QAEH;;;;WAIG;;;;;;;;;;;CASqB,CAAC"}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The alias-resolving LLM client.
|
|
3
|
+
*
|
|
4
|
+
* This is the only supported way to reach a language model from application
|
|
5
|
+
* code. Its signature mirrors the incumbent client's — content, response
|
|
6
|
+
* format, options — so migrating a call site is replacing a model string with
|
|
7
|
+
* an alias, and nothing else. That similarity is the point: a migration that
|
|
8
|
+
* required rewriting call sites would be a migration that stalls half-done,
|
|
9
|
+
* leaving some calls inside the timeout, breaker and fallback controls and
|
|
10
|
+
* some outside them, which is worse than either end state.
|
|
11
|
+
*
|
|
12
|
+
* There is deliberately no way to name a model. PD-5 makes vendor strings a CI
|
|
13
|
+
* failure in application code, and an option that accepted one would let a call
|
|
14
|
+
* site opt out of the routing policy without anyone noticing.
|
|
15
|
+
*
|
|
16
|
+
* Every call gets, in order: alias resolution against the canonical route
|
|
17
|
+
* table, per-provider parameter normalisation, a hard per-leg timeout, a
|
|
18
|
+
* per-route circuit breaker, an ordered fallback chain ending at the closed
|
|
19
|
+
* incumbent, and — where the caller supplies a validator — one schema-feedback
|
|
20
|
+
* retry ahead of the chain. None of them is optional, because a control that a
|
|
21
|
+
* caller can switch off is a control that will be off on the call that needed
|
|
22
|
+
* it.
|
|
23
|
+
*
|
|
24
|
+
* @module llm/alias-client
|
|
25
|
+
*/
|
|
26
|
+
import { CircuitBreakerRegistry } from "./circuit-breaker";
|
|
27
|
+
import type { AliasCallOptions, AliasCallResult, LlmAlias, LlmClientConfig, LlmResponseFormat } from "./types";
|
|
28
|
+
/**
|
|
29
|
+
* Wire the client.
|
|
30
|
+
*
|
|
31
|
+
* Called once at process start. Transports are injectable so a consumer can
|
|
32
|
+
* supply its own instrumented client, and so tests can exercise the chain
|
|
33
|
+
* without a network — a fallback chain that could only be observed against live
|
|
34
|
+
* providers would in practice never be observed at all.
|
|
35
|
+
*
|
|
36
|
+
* @param next Runtime wiring; unspecified fields fall back to the environment.
|
|
37
|
+
* @returns void
|
|
38
|
+
*/
|
|
39
|
+
export declare function configureLlmClient(next: LlmClientConfig): void;
|
|
40
|
+
/**
|
|
41
|
+
* Inspect route health.
|
|
42
|
+
*
|
|
43
|
+
* @returns The live breaker registry.
|
|
44
|
+
*/
|
|
45
|
+
export declare function llmBreakers(): CircuitBreakerRegistry;
|
|
46
|
+
/**
|
|
47
|
+
* Call a language model by semantic alias.
|
|
48
|
+
*
|
|
49
|
+
* @param content The prompt, or multi-part content for a vision-capable route.
|
|
50
|
+
* @param responseFormat The response shape. Defaults to plain text.
|
|
51
|
+
* @param options Call options; `alias` is required and there is no model option.
|
|
52
|
+
* @returns The answer, with the routing decision and full attempt record attached.
|
|
53
|
+
* @throws {UnknownAliasError} When the alias is not in the route table.
|
|
54
|
+
* @throws {ChainExhaustedError} When no leg produced an answer.
|
|
55
|
+
* @throws {SchemaRetryExhaustedError} When a validated payload failed twice.
|
|
56
|
+
*/
|
|
57
|
+
export declare function callLLMByAlias<T = unknown>(content: string | readonly unknown[], responseFormat: LlmResponseFormat | undefined, options: AliasCallOptions<T>): Promise<AliasCallResult<T>>;
|
|
58
|
+
/**
|
|
59
|
+
* The aliases application code may name.
|
|
60
|
+
*
|
|
61
|
+
* @returns The alias names, sorted.
|
|
62
|
+
*/
|
|
63
|
+
export declare function llmAliases(): LlmAlias[];
|
|
64
|
+
//# sourceMappingURL=alias-client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"alias-client.d.ts","sourceRoot":"","sources":["../../../src/llm/alias-client.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAY3D,OAAO,KAAK,EAEV,gBAAgB,EAChB,eAAe,EACf,QAAQ,EACR,eAAe,EACf,iBAAiB,EAGlB,MAAM,SAAS,CAAC;AAoBjB;;;;;;;;;;GAUG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,eAAe,GAAG,IAAI,CAQ9D;AAED;;;;GAIG;AACH,wBAAgB,WAAW,IAAI,sBAAsB,CAEpD;AA+ED;;;;;;;;;;GAUG;AACH,wBAAsB,cAAc,CAAC,CAAC,GAAG,OAAO,EAC9C,OAAO,EAAE,MAAM,GAAG,SAAS,OAAO,EAAE,EACpC,cAAc,EAAE,iBAAiB,YAAS,EAC1C,OAAO,EAAE,gBAAgB,CAAC,CAAC,CAAC,GAC3B,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAoK7B;AA8BD;;;;GAIG;AACH,wBAAgB,UAAU,IAAI,QAAQ,EAAE,CAEvC"}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-route circuit breaker for the alias client.
|
|
3
|
+
*
|
|
4
|
+
* A provider that has just failed five calls will almost certainly fail the
|
|
5
|
+
* sixth, and every attempt spends latency budget the next leg of the chain
|
|
6
|
+
* needs. The breaker converts that repeated discovery into a decision made
|
|
7
|
+
* once: an unhealthy leg is skipped outright until it has had time to recover,
|
|
8
|
+
* so a chain reaches its working leg quickly instead of paying the full
|
|
9
|
+
* timeout of each dead one first.
|
|
10
|
+
*
|
|
11
|
+
* Breakers are keyed by chain POSITION, not by provider. Two consequences
|
|
12
|
+
* follow, and both are intended. Reverting an alias to a different model at the
|
|
13
|
+
* same position inherits that position's health rather than starting blind. And
|
|
14
|
+
* an isolated route never shares a breaker with the shared one, so isolated
|
|
15
|
+
* traffic can neither trip nor be tripped by traffic on the other side of the
|
|
16
|
+
* PD-9 boundary.
|
|
17
|
+
*
|
|
18
|
+
* The clock is injected. Breaker behaviour is entirely about elapsed time, and
|
|
19
|
+
* a test that must sleep to observe a cooldown is a test nobody runs.
|
|
20
|
+
*
|
|
21
|
+
* @module llm/circuit-breaker
|
|
22
|
+
*/
|
|
23
|
+
import type { LlmBreakerDefaults } from "./types";
|
|
24
|
+
/** What the breaker will currently permit for a route. */
|
|
25
|
+
export type BreakerState = "closed" | "open" | "half-open";
|
|
26
|
+
/** Snapshot of one route's breaker, for observability and tests. */
|
|
27
|
+
export interface BreakerSnapshot {
|
|
28
|
+
readonly routeKey: string;
|
|
29
|
+
readonly state: BreakerState;
|
|
30
|
+
readonly consecutiveFailures: number;
|
|
31
|
+
readonly openedAtMs: number | null;
|
|
32
|
+
readonly probesInFlight: number;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Tracks route health and decides whether a leg may be attempted.
|
|
36
|
+
*/
|
|
37
|
+
export declare class CircuitBreakerRegistry {
|
|
38
|
+
private readonly records;
|
|
39
|
+
private readonly config;
|
|
40
|
+
private readonly now;
|
|
41
|
+
/**
|
|
42
|
+
* @param config Failure threshold, cooldown and half-open probe budget.
|
|
43
|
+
* @param now Clock, injected so cooldowns are testable without waiting.
|
|
44
|
+
*/
|
|
45
|
+
constructor(config: LlmBreakerDefaults, now?: () => number);
|
|
46
|
+
/**
|
|
47
|
+
* Current state of a route's breaker.
|
|
48
|
+
*
|
|
49
|
+
* The transition from open to half-open is computed from elapsed time at read
|
|
50
|
+
* time rather than scheduled with a timer. A timer would keep the process
|
|
51
|
+
* awake for every route that ever failed, and would drift whenever the
|
|
52
|
+
* process was busy — which is exactly when a breaker matters most.
|
|
53
|
+
*
|
|
54
|
+
* @param routeKey The route's stable key.
|
|
55
|
+
* @returns Its state.
|
|
56
|
+
*/
|
|
57
|
+
stateOf(routeKey: string): BreakerState;
|
|
58
|
+
/**
|
|
59
|
+
* Whether a route may be attempted now.
|
|
60
|
+
*
|
|
61
|
+
* A half-open route admits a bounded number of probes at once. Letting the
|
|
62
|
+
* whole queue through the moment a cooldown expires would re-hammer a
|
|
63
|
+
* provider that is still recovering, which is how a breaker turns into a
|
|
64
|
+
* synchronised retry storm.
|
|
65
|
+
*
|
|
66
|
+
* @param routeKey The route's stable key.
|
|
67
|
+
* @returns Whether an attempt is permitted.
|
|
68
|
+
*/
|
|
69
|
+
allows(routeKey: string): boolean;
|
|
70
|
+
/**
|
|
71
|
+
* Register that an attempt is starting, so half-open probes stay bounded.
|
|
72
|
+
*
|
|
73
|
+
* @param routeKey The route's stable key.
|
|
74
|
+
* @returns void
|
|
75
|
+
*/
|
|
76
|
+
onAttemptStart(routeKey: string): void;
|
|
77
|
+
/**
|
|
78
|
+
* Record a success, closing the breaker.
|
|
79
|
+
*
|
|
80
|
+
* A single success closes it fully rather than decrementing the failure
|
|
81
|
+
* count. The breaker's question is "is this route working now", and one
|
|
82
|
+
* working call answers it; requiring several would keep a recovered provider
|
|
83
|
+
* excluded while the chain paid for slower legs.
|
|
84
|
+
*
|
|
85
|
+
* @param routeKey The route's stable key.
|
|
86
|
+
* @returns void
|
|
87
|
+
*/
|
|
88
|
+
onSuccess(routeKey: string): void;
|
|
89
|
+
/**
|
|
90
|
+
* Record a failure, opening the breaker once the threshold is reached.
|
|
91
|
+
*
|
|
92
|
+
* A failure while half-open re-opens immediately without waiting to
|
|
93
|
+
* re-accumulate the threshold: the probe was the test, and it failed.
|
|
94
|
+
*
|
|
95
|
+
* @param routeKey The route's stable key.
|
|
96
|
+
* @returns void
|
|
97
|
+
*/
|
|
98
|
+
onFailure(routeKey: string): void;
|
|
99
|
+
/**
|
|
100
|
+
* Inspect a route's breaker.
|
|
101
|
+
*
|
|
102
|
+
* @param routeKey The route's stable key.
|
|
103
|
+
* @returns A snapshot.
|
|
104
|
+
*/
|
|
105
|
+
snapshot(routeKey: string): BreakerSnapshot;
|
|
106
|
+
/**
|
|
107
|
+
* Every route the registry has observed.
|
|
108
|
+
*
|
|
109
|
+
* @returns Snapshots, sorted by route key for stable output.
|
|
110
|
+
*/
|
|
111
|
+
snapshotAll(): BreakerSnapshot[];
|
|
112
|
+
/**
|
|
113
|
+
* Discard all breaker state.
|
|
114
|
+
*
|
|
115
|
+
* @returns void
|
|
116
|
+
*/
|
|
117
|
+
reset(): void;
|
|
118
|
+
/**
|
|
119
|
+
* @param routeKey The route's stable key.
|
|
120
|
+
* @returns The mutable record, created on first use.
|
|
121
|
+
*/
|
|
122
|
+
private recordFor;
|
|
123
|
+
}
|
|
124
|
+
//# sourceMappingURL=circuit-breaker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"circuit-breaker.d.ts","sourceRoot":"","sources":["../../../src/llm/circuit-breaker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAElD,0DAA0D;AAC1D,MAAM,MAAM,YAAY,GAAG,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;AAE3D,oEAAoE;AACpE,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC;AAQD;;GAEG;AACH,qBAAa,sBAAsB;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoC;IAE5D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;IAE5C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IAEnC;;;OAGG;gBACgB,MAAM,EAAE,kBAAkB,EAAE,GAAG,GAAE,MAAM,MAAiB;IAK3E;;;;;;;;;;OAUG;IACI,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,YAAY;IAS9C;;;;;;;;;;OAUG;IACI,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAYxC;;;;;OAKG;IACI,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAM7C;;;;;;;;;;OAUG;IACI,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAQxC;;;;;;;;OAQG;IACI,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAWxC;;;;;OAKG;IACI,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,eAAe;IAelD;;;;OAIG;IACI,WAAW,IAAI,eAAe,EAAE;IAMvC;;;;OAIG;IACI,KAAK,IAAI,IAAI;IAIpB;;;OAGG;IACH,OAAO,CAAC,SAAS;CAQlB"}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The incumbent-relative comparators, one per assertion type in Section 6 of
|
|
3
|
+
* the migration backlog.
|
|
4
|
+
*
|
|
5
|
+
* Each is a pure function from two measured numbers to a verdict, which is what
|
|
6
|
+
* makes PD-6 enforceable: the gate is decided by code from recorded data, and
|
|
7
|
+
* there is no place in the call for judgement to enter. Keeping them separate
|
|
8
|
+
* and separately testable also keeps their tolerances honest — a tolerance
|
|
9
|
+
* buried inside a runner is a tolerance nobody re-reads.
|
|
10
|
+
*
|
|
11
|
+
* Three preconditions are shared by all of them, and all three yield
|
|
12
|
+
* `indeterminate` rather than a verdict:
|
|
13
|
+
*
|
|
14
|
+
* - no recorded incumbent value, because every gate here is a comparison and a
|
|
15
|
+
* comparison with one side missing is not a strict measurement;
|
|
16
|
+
* - no candidate value, for the same reason;
|
|
17
|
+
* - fewer cases than the set's declared minimum, because a difference measured
|
|
18
|
+
* on too few cases is indistinguishable from sampling noise.
|
|
19
|
+
*
|
|
20
|
+
* @module llm/eval/comparators
|
|
21
|
+
*/
|
|
22
|
+
import type { EvalAssertion, EvalGate, Verdict } from "./types";
|
|
23
|
+
/**
|
|
24
|
+
* Tolerance for the match assertion, in points.
|
|
25
|
+
*
|
|
26
|
+
* Section 6: "exact-match or field-level F1 within 1 point of incumbent". The
|
|
27
|
+
* band is two-sided in principle and one-sided in effect — a candidate above
|
|
28
|
+
* the incumbent passes outright — so only the downside is checked.
|
|
29
|
+
*/
|
|
30
|
+
export declare const MATCH_TOLERANCE_POINTS = 1;
|
|
31
|
+
/**
|
|
32
|
+
* Tolerance for the pinned-judge assertion, as a fraction of the incumbent's score.
|
|
33
|
+
*
|
|
34
|
+
* Section 6: "pinned-judge score within 3% of incumbent". Relative rather than
|
|
35
|
+
* absolute because judge scores are not linear across their range: three points
|
|
36
|
+
* near the top of the scale is a much larger regression than three points in
|
|
37
|
+
* the middle.
|
|
38
|
+
*/
|
|
39
|
+
export declare const JUDGE_TOLERANCE_FRACTION = 0.03;
|
|
40
|
+
/** Scale factor converting a 0-1 rate into the 0-100 point scale tolerances are stated in. */
|
|
41
|
+
export declare const RATE_TO_POINTS = 100;
|
|
42
|
+
/** The measured inputs one comparator decides on. */
|
|
43
|
+
export interface ComparatorInput {
|
|
44
|
+
/** The candidate's measured value, or `undefined` when it could not be measured. */
|
|
45
|
+
readonly candidate: number | undefined;
|
|
46
|
+
/** The recorded incumbent value, or `undefined` when the set has no baseline for it. */
|
|
47
|
+
readonly incumbent: number | undefined;
|
|
48
|
+
/** Cases the measurement was taken over. */
|
|
49
|
+
readonly n: number;
|
|
50
|
+
/** The set's declared minimum case count. */
|
|
51
|
+
readonly minN: number;
|
|
52
|
+
}
|
|
53
|
+
/** A comparator: measured inputs in, one verdict out. */
|
|
54
|
+
export type Comparator = (input: ComparatorInput) => Verdict;
|
|
55
|
+
/**
|
|
56
|
+
* Schema-valid rate: the candidate must parse at least as often as the incumbent.
|
|
57
|
+
*
|
|
58
|
+
* No tolerance band, because a malformed structured answer is not a slightly
|
|
59
|
+
* worse answer — it is one the call site cannot consume at all.
|
|
60
|
+
*
|
|
61
|
+
* @param input The measured rates, on a 0-1 scale.
|
|
62
|
+
* @returns The verdict.
|
|
63
|
+
*/
|
|
64
|
+
export declare const compareSchemaValidRate: Comparator;
|
|
65
|
+
/**
|
|
66
|
+
* Tool-call valid-call rate: the candidate must call correctly at least as often.
|
|
67
|
+
*
|
|
68
|
+
* The permitted single structured retry is applied when the rate is derived, so
|
|
69
|
+
* by the time the comparison is made both sides have been measured under the
|
|
70
|
+
* same retry budget.
|
|
71
|
+
*
|
|
72
|
+
* @param input The measured rates, on a 0-1 scale.
|
|
73
|
+
* @returns The verdict.
|
|
74
|
+
*/
|
|
75
|
+
export declare const compareValidCallRate: Comparator;
|
|
76
|
+
/**
|
|
77
|
+
* Exact-match or field-level F1: the candidate must stay within one point.
|
|
78
|
+
*
|
|
79
|
+
* Both measurements are expressed in points on a 0-100 scale before the
|
|
80
|
+
* comparison, so the same tolerance means the same thing whichever a set is
|
|
81
|
+
* scored on.
|
|
82
|
+
*
|
|
83
|
+
* @param input The measured scores, in points.
|
|
84
|
+
* @returns The verdict.
|
|
85
|
+
*/
|
|
86
|
+
export declare const compareMatchScore: Comparator;
|
|
87
|
+
/**
|
|
88
|
+
* Pinned-judge score: the candidate must stay within 3% of the incumbent.
|
|
89
|
+
*
|
|
90
|
+
* The comparison assumes the pinning guard has already run; a judged verdict
|
|
91
|
+
* produced by an unpinned judge would grade one model's output against another
|
|
92
|
+
* model's standard, which is why the guard refuses rather than warns.
|
|
93
|
+
*
|
|
94
|
+
* @param input The measured scores, in points.
|
|
95
|
+
* @returns The verdict.
|
|
96
|
+
*/
|
|
97
|
+
export declare const compareJudgeScore: Comparator;
|
|
98
|
+
/**
|
|
99
|
+
* Latency: the candidate's p95 must not exceed the incumbent's.
|
|
100
|
+
*
|
|
101
|
+
* Lower is better here, which is why this is the one comparator that cannot
|
|
102
|
+
* share the "at least as good" helper — inverting a comparison by reusing a
|
|
103
|
+
* helper with swapped arguments is exactly the kind of quiet sign error a
|
|
104
|
+
* latency gate would never surface on its own.
|
|
105
|
+
*
|
|
106
|
+
* @param input The measured p95 values, in milliseconds.
|
|
107
|
+
* @returns The verdict.
|
|
108
|
+
*/
|
|
109
|
+
export declare const compareLatencyP95: Comparator;
|
|
110
|
+
/**
|
|
111
|
+
* Every assertion type mapped to the comparator that decides it.
|
|
112
|
+
*
|
|
113
|
+
* A registry rather than a switch, so "is every assertion type implemented" is
|
|
114
|
+
* a question code can answer — which is what the harness's own coverage check
|
|
115
|
+
* asks before it will certify anything.
|
|
116
|
+
*/
|
|
117
|
+
export declare const COMPARATORS: Readonly<Record<EvalAssertion, Comparator>>;
|
|
118
|
+
/**
|
|
119
|
+
* The assertions each eval-gate category requires.
|
|
120
|
+
*
|
|
121
|
+
* `none-pinned-judge` requires none by design: the judge alias is the
|
|
122
|
+
* instrument, not a subject. It is held to the pinning guard instead, which is
|
|
123
|
+
* a stricter contract than any score comparison — the judge may not change at
|
|
124
|
+
* all, rather than merely not getting worse.
|
|
125
|
+
*/
|
|
126
|
+
export declare const EVAL_GATE_ASSERTIONS: Readonly<Record<EvalGate, readonly EvalAssertion[]>>;
|
|
127
|
+
//# sourceMappingURL=comparators.d.ts.map
|