@mcp-abap-adt/llm-agent 22.2.0 → 24.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/interfaces/index.d.ts +1 -0
- package/dist/interfaces/index.d.ts.map +1 -1
- package/dist/interfaces/index.js +1 -0
- package/dist/interfaces/index.js.map +1 -1
- package/dist/interfaces/throttle-strategy.d.ts +103 -0
- package/dist/interfaces/throttle-strategy.d.ts.map +1 -0
- package/dist/interfaces/throttle-strategy.js +86 -0
- package/dist/interfaces/throttle-strategy.js.map +1 -0
- package/dist/llm/base-llm-provider.d.ts +18 -10
- package/dist/llm/base-llm-provider.d.ts.map +1 -1
- package/dist/llm/base-llm-provider.js +32 -20
- package/dist/llm/base-llm-provider.js.map +1 -1
- package/dist/llm/throttle.d.ts +218 -0
- package/dist/llm/throttle.d.ts.map +1 -0
- package/dist/llm/throttle.js +441 -0
- package/dist/llm/throttle.js.map +1 -0
- package/dist/types.d.ts +8 -6
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -5,7 +5,7 @@ export * from './external-results.js';
|
|
|
5
5
|
export { assertClientDescriptors } from './interfaces/assert-client-descriptors.js';
|
|
6
6
|
export * from './interfaces/index.js';
|
|
7
7
|
export { BaseLLMProvider, type LLMProvider } from './llm/base-llm-provider.js';
|
|
8
|
-
export {
|
|
8
|
+
export { findThrottled, gateFor, isThrottledError, preserveThrottled, pruneQuotaGates, QuotaGate, quotaGateCount, resetQuotaGates, runWithThrottleRetry, setThrottleObserver, type ThrottledError, type ThrottleEvent, type ThrottleRetryOptions, } from './llm/throttle.js';
|
|
9
9
|
export { NeedsDecompositionError } from './needs-decomposition-error.js';
|
|
10
10
|
export * from './rag/index.js';
|
|
11
11
|
export * from './types.js';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EACL,mBAAmB,EACnB,iBAAiB,EACjB,kBAAkB,EAClB,SAAS,EACT,aAAa,GACd,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AACzE,cAAc,mBAAmB,CAAC;AAClC,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,uBAAuB,EAAE,MAAM,2CAA2C,CAAC;AACpF,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,KAAK,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAC/E,OAAO,EACL,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EACL,mBAAmB,EACnB,iBAAiB,EACjB,kBAAkB,EAClB,SAAS,EACT,aAAa,GACd,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AACzE,cAAc,mBAAmB,CAAC;AAClC,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,uBAAuB,EAAE,MAAM,2CAA2C,CAAC;AACpF,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,KAAK,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAC/E,OAAO,EACL,aAAa,EACb,OAAO,EACP,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,SAAS,EACT,cAAc,EACd,eAAe,EACf,oBAAoB,EACpB,mBAAmB,EACnB,KAAK,cAAc,EACnB,KAAK,aAAa,EAClB,KAAK,oBAAoB,GAC1B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AACzE,cAAc,gBAAgB,CAAC;AAC/B,cAAc,YAAY,CAAC;AAI3B,OAAO,EAAE,kBAAkB,EAAE,MAAM,oCAAoC,CAAC;AAExE,OAAO,EAAE,mBAAmB,EAAE,MAAM,qCAAqC,CAAC;AAC1E,OAAO,EAAE,gBAAgB,EAAE,MAAM,kCAAkC,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAElD,YAAY,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,EACL,sBAAsB,EACtB,KAAK,0BAA0B,EAC/B,KAAK,2BAA2B,EAChC,iCAAiC,EACjC,sBAAsB,GACvB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EACL,sBAAsB,EACtB,KAAK,iBAAiB,EACtB,KAAK,WAAW,EAChB,KAAK,cAAc,EACnB,KAAK,iBAAiB,GACvB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACL,oBAAoB,EACpB,KAAK,oBAAoB,GAC1B,MAAM,wCAAwC,CAAC;AAEhD,YAAY,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AACrE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAC;AACxE,OAAO,EACL,gBAAgB,EAChB,oBAAoB,EACpB,KAAK,cAAc,EACnB,KAAK,oBAAoB,GAC1B,MAAM,gCAAgC,CAAC;AAExC,YAAY,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC3D,OAAO,EAAE,uBAAuB,EAAE,MAAM,wCAAwC,CAAC;AAEjF,OAAO,EAAE,2BAA2B,EAAE,MAAM,6CAA6C,CAAC;AAC1F,OAAO,EAAE,wBAAwB,EAAE,MAAM,yCAAyC,CAAC;AACnF,OAAO,EACL,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,yCAAyC,CAAC;AAEjD,OAAO,EACL,cAAc,EACd,KAAK,oBAAoB,EACzB,KAAK,YAAY,GAClB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EACL,sBAAsB,EACtB,0BAA0B,EAC1B,kBAAkB,GACnB,MAAM,0CAA0C,CAAC;AAClD,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAC;AACxE,OAAO,EACL,cAAc,EACd,KAAK,wBAAwB,EAC7B,wBAAwB,EACxB,KAAK,0BAA0B,EAC/B,qBAAqB,EACrB,eAAe,GAChB,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAC3D,OAAO,EACL,KAAK,oBAAoB,EACzB,iBAAiB,EACjB,iBAAiB,EACjB,kBAAkB,EAClB,aAAa,EACb,SAAS,GACV,MAAM,gCAAgC,CAAC;AACxC,OAAO,EACL,qBAAqB,EACrB,eAAe,GAChB,MAAM,uBAAuB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -6,7 +6,7 @@ export * from './external-results.js';
|
|
|
6
6
|
export { assertClientDescriptors } from './interfaces/assert-client-descriptors.js';
|
|
7
7
|
export * from './interfaces/index.js';
|
|
8
8
|
export { BaseLLMProvider } from './llm/base-llm-provider.js';
|
|
9
|
-
export {
|
|
9
|
+
export { findThrottled, gateFor, isThrottledError, preserveThrottled, pruneQuotaGates, QuotaGate, quotaGateCount, resetQuotaGates, runWithThrottleRetry, setThrottleObserver, } from './llm/throttle.js';
|
|
10
10
|
export { NeedsDecompositionError } from './needs-decomposition-error.js';
|
|
11
11
|
export * from './rag/index.js';
|
|
12
12
|
export * from './types.js';
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,2CAA2C;AAC3C,OAAO,EACL,mBAAmB,EACnB,iBAAiB,EACjB,kBAAkB,EAClB,SAAS,EACT,aAAa,GACd,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AACzE,cAAc,mBAAmB,CAAC;AAClC,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,uBAAuB,EAAE,MAAM,2CAA2C,CAAC;AACpF,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,eAAe,EAAoB,MAAM,4BAA4B,CAAC;AAC/E,OAAO,EACL,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,2CAA2C;AAC3C,OAAO,EACL,mBAAmB,EACnB,iBAAiB,EACjB,kBAAkB,EAClB,SAAS,EACT,aAAa,GACd,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AACzE,cAAc,mBAAmB,CAAC;AAClC,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,uBAAuB,EAAE,MAAM,2CAA2C,CAAC;AACpF,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,eAAe,EAAoB,MAAM,4BAA4B,CAAC;AAC/E,OAAO,EACL,aAAa,EACb,OAAO,EACP,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,SAAS,EACT,cAAc,EACd,eAAe,EACf,oBAAoB,EACpB,mBAAmB,GAIpB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AACzE,cAAc,gBAAgB,CAAC;AAC/B,cAAc,YAAY,CAAC;AAE3B,sEAAsE;AAEtE,OAAO,EAAE,kBAAkB,EAAE,MAAM,oCAAoC,CAAC;AACxE,eAAe;AACf,OAAO,EAAE,mBAAmB,EAAE,MAAM,qCAAqC,CAAC;AAC1E,OAAO,EAAE,gBAAgB,EAAE,MAAM,kCAAkC,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAGlD,iBAAiB;AACjB,OAAO,EACL,sBAAsB,EAGtB,iCAAiC,EACjC,sBAAsB,GACvB,MAAM,gCAAgC,CAAC;AACxC,yBAAyB;AACzB,OAAO,EACL,sBAAsB,GAKvB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACL,oBAAoB,GAErB,MAAM,wCAAwC,CAAC;AAGhD,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAC;AACxE,OAAO,EACL,gBAAgB,EAChB,oBAAoB,GAGrB,MAAM,gCAAgC,CAAC;AAGxC,OAAO,EAAE,uBAAuB,EAAE,MAAM,wCAAwC,CAAC;AACjF,oBAAoB;AACpB,OAAO,EAAE,2BAA2B,EAAE,MAAM,6CAA6C,CAAC;AAC1F,OAAO,EAAE,wBAAwB,EAAE,MAAM,yCAAyC,CAAC;AACnF,OAAO,EACL,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,yCAAyC,CAAC;AACjD,aAAa;AACb,OAAO,EACL,cAAc,GAGf,MAAM,iCAAiC,CAAC;AACzC,OAAO,EACL,sBAAsB,EACtB,0BAA0B,EAC1B,kBAAkB,GACnB,MAAM,0CAA0C,CAAC;AAClD,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAC;AACxE,OAAO,EACL,cAAc,EAEd,wBAAwB,EAExB,qBAAqB,EACrB,eAAe,GAChB,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAC3D,OAAO,EAEL,iBAAiB,EACjB,iBAAiB,EACjB,kBAAkB,EAClB,aAAa,EACb,SAAS,GACV,MAAM,gCAAgC,CAAC;AACxC,OAAO,EACL,qBAAqB,EACrB,eAAe,GAChB,MAAM,uBAAuB,CAAC"}
|
|
@@ -56,6 +56,7 @@ export type { ISubAgent, ISubAgentInput, ISubAgentResult, SubAgentCapabilities,
|
|
|
56
56
|
export type { ISubAgentContextBuilder, SubAgentContextRequest, SubAgentContextResult, } from './subagent-context.js';
|
|
57
57
|
export type { ITaskFormalizer, ITaskSpec } from './task-spec.js';
|
|
58
58
|
export { renderTaskSpec } from './task-spec.js';
|
|
59
|
+
export { type IThrottleStrategy, ReportThrottling, type ThrottleContext, type ThrottleDecision, WaitAsTold, } from './throttle-strategy.js';
|
|
59
60
|
export { type IToolCatalogReporter, isToolCatalogReporter, type ToolCatalogStatus, } from './tool-catalog.js';
|
|
60
61
|
export type { IToolLoopContextStrategy, JsonValue, SerializableStrategyState, ToolLoopContextBase, ToolLoopContextStrategyDeps, ToolLoopContextStrategyFactory, ToolRound, } from './tool-loop-context-strategy.js';
|
|
61
62
|
export { defaultToolRecordKey, type IToolRecordKey, type ToolKeyContext, toolNameFromRecord, } from './tool-record-key.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/interfaces/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,kBAAkB,EACvB,iBAAiB,EACjB,KAAK,kBAAkB,EACvB,KAAK,UAAU,GAChB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACtE,YAAY,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AACnE,YAAY,EACV,WAAW,EACX,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,cAAc,CAAC;AACtB,YAAY,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAC5D,YAAY,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,YAAY,EACV,aAAa,EACb,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,iBAAiB,EACjB,iBAAiB,EACjB,IAAI,EACJ,QAAQ,EACR,UAAU,EACV,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACvD,YAAY,EACV,YAAY,EACZ,aAAa,EACb,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EACV,cAAc,EACd,gBAAgB,EAChB,UAAU,GACX,MAAM,gBAAgB,CAAC;AACxB,YAAY,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC/C,YAAY,EACV,cAAc,EACd,eAAe,EACf,UAAU,GACX,MAAM,gBAAgB,CAAC;AACxB,YAAY,EACV,oBAAoB,EACpB,qBAAqB,EACrB,YAAY,GACb,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,YAAY,EACV,WAAW,EACX,kBAAkB,GACnB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,YAAY,EACV,YAAY,EACZ,gBAAgB,EAChB,eAAe,EACf,UAAU,GACX,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,mBAAmB,EACnB,eAAe,EACf,cAAc,EACd,sBAAsB,EACtB,eAAe,GAChB,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AACrC,YAAY,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC/D,YAAY,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAClD,YAAY,EACV,yBAAyB,EACzB,sBAAsB,EACtB,mBAAmB,EACnB,gBAAgB,EAChB,sBAAsB,EACtB,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,8BAA8B,CAAC;AACtC,YAAY,EACV,qBAAqB,EACrB,cAAc,GACf,MAAM,6BAA6B,CAAC;AACrC,YAAY,EAAE,0BAA0B,EAAE,MAAM,mCAAmC,CAAC;AACpF,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,QAAQ,EACR,UAAU,EACV,QAAQ,EACR,eAAe,GAChB,MAAM,cAAc,CAAC;AACtB,YAAY,EACV,YAAY,EACZ,UAAU,EACV,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,YAAY,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACxD,YAAY,EACV,gBAAgB,EAChB,eAAe,EACf,eAAe,EACf,SAAS,GACV,MAAM,eAAe,CAAC;AACvB,YAAY,EACV,gBAAgB,EAChB,gBAAgB,EAChB,uBAAuB,EACvB,mBAAmB,GACpB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EACV,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,yBAAyB,EACzB,YAAY,GACb,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,QAAQ,EACR,mBAAmB,EACnB,YAAY,EACZ,aAAa,GACd,MAAM,cAAc,CAAC;AACtB,YAAY,EACV,aAAa,EACb,eAAe,EACf,iBAAiB,EACjB,aAAa,EACb,aAAa,EACb,aAAa,EACb,QAAQ,GACT,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,YAAY,EACV,eAAe,EACf,qBAAqB,EACrB,SAAS,EACT,cAAc,EACd,YAAY,EACZ,WAAW,EACX,IAAI,EACJ,iBAAiB,EACjB,UAAU,EACV,YAAY,EACZ,oBAAoB,EACpB,YAAY,EACZ,iBAAiB,EACjB,kBAAkB,GACnB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC/D,YAAY,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EACL,KAAK,kBAAkB,EACvB,mBAAmB,GACpB,MAAM,yBAAyB,CAAC;AACjC,YAAY,EACV,cAAc,EACd,YAAY,EACZ,YAAY,EACZ,aAAa,EACb,cAAc,EACd,WAAW,EACX,aAAa,EACb,aAAa,GACd,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC/C,YAAY,EACV,qBAAqB,EACrB,uBAAuB,EACvB,qBAAqB,EACrB,eAAe,EACf,WAAW,EACX,YAAY,EACZ,aAAa,GACd,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpD,YAAY,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7D,YAAY,EACV,MAAM,EACN,aAAa,EACb,UAAU,EACV,cAAc,GACf,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,cAAc,EACd,YAAY,EACZ,eAAe,EACf,gBAAgB,EAChB,YAAY,EACZ,cAAc,EACd,iBAAiB,EACjB,yBAAyB,EACzB,gBAAgB,EAChB,YAAY,EACZ,oBAAoB,EACpB,iBAAiB,EACjB,cAAc,EACd,QAAQ,EACR,iBAAiB,EACjB,eAAe,EACf,WAAW,EACX,yBAAyB,EACzB,cAAc,GACf,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,eAAe,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AAC3E,YAAY,EACV,YAAY,EACZ,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EACV,UAAU,EACV,oBAAoB,EACpB,WAAW,EACX,qBAAqB,EACrB,iBAAiB,EACjB,QAAQ,EACR,eAAe,EACf,kBAAkB,EAClB,mBAAmB,EACnB,cAAc,GACf,MAAM,6BAA6B,CAAC;AACrC,YAAY,EACV,MAAM,EACN,QAAQ,EACR,aAAa,EACb,cAAc,EACd,YAAY,EACZ,WAAW,GACZ,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,YAAY,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,YAAY,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,YAAY,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AACzE,YAAY,EACV,SAAS,EACT,cAAc,EACd,eAAe,EACf,oBAAoB,EACpB,gBAAgB,GACjB,MAAM,eAAe,CAAC;AACvB,YAAY,EACV,uBAAuB,EACvB,sBAAsB,EACtB,qBAAqB,GACtB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AACjE,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EACL,KAAK,oBAAoB,EACzB,qBAAqB,EACrB,KAAK,iBAAiB,GACvB,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EACV,wBAAwB,EACxB,SAAS,EACT,yBAAyB,EACzB,mBAAmB,EACnB,2BAA2B,EAC3B,8BAA8B,EAC9B,SAAS,GACV,MAAM,iCAAiC,CAAC;AACzC,OAAO,EACL,oBAAoB,EACpB,KAAK,cAAc,EACnB,KAAK,cAAc,EACnB,kBAAkB,GACnB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAE,sBAAsB,EAAE,MAAM,8BAA8B,CAAC;AAC3E,YAAY,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC3E,YAAY,EACV,WAAW,EACX,WAAW,EACX,YAAY,EACZ,eAAe,EACf,WAAW,EACX,cAAc,EACd,OAAO,EACP,WAAW,EACX,gBAAgB,EAChB,QAAQ,EACR,aAAa,EACb,eAAe,EACf,OAAO,EACP,aAAa,EACb,eAAe,EACf,WAAW,EACX,SAAS,EACT,MAAM,EACN,iBAAiB,EACjB,cAAc,EACd,SAAS,EACT,aAAa,EACb,WAAW,EACX,cAAc,EACd,aAAa,EACb,YAAY,GACb,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,cAAc,EACd,eAAe,EACf,gBAAgB,EAChB,QAAQ,EACR,qBAAqB,EACrB,QAAQ,EACR,QAAQ,EACR,UAAU,EACV,eAAe,GAChB,MAAM,YAAY,CAAC;AACpB,YAAY,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,YAAY,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/interfaces/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,kBAAkB,EACvB,iBAAiB,EACjB,KAAK,kBAAkB,EACvB,KAAK,UAAU,GAChB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACtE,YAAY,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AACnE,YAAY,EACV,WAAW,EACX,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,cAAc,CAAC;AACtB,YAAY,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAC5D,YAAY,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,YAAY,EACV,aAAa,EACb,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,iBAAiB,EACjB,iBAAiB,EACjB,IAAI,EACJ,QAAQ,EACR,UAAU,EACV,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACvD,YAAY,EACV,YAAY,EACZ,aAAa,EACb,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EACV,cAAc,EACd,gBAAgB,EAChB,UAAU,GACX,MAAM,gBAAgB,CAAC;AACxB,YAAY,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC/C,YAAY,EACV,cAAc,EACd,eAAe,EACf,UAAU,GACX,MAAM,gBAAgB,CAAC;AACxB,YAAY,EACV,oBAAoB,EACpB,qBAAqB,EACrB,YAAY,GACb,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,YAAY,EACV,WAAW,EACX,kBAAkB,GACnB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,YAAY,EACV,YAAY,EACZ,gBAAgB,EAChB,eAAe,EACf,UAAU,GACX,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,mBAAmB,EACnB,eAAe,EACf,cAAc,EACd,sBAAsB,EACtB,eAAe,GAChB,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AACrC,YAAY,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC/D,YAAY,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAClD,YAAY,EACV,yBAAyB,EACzB,sBAAsB,EACtB,mBAAmB,EACnB,gBAAgB,EAChB,sBAAsB,EACtB,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,8BAA8B,CAAC;AACtC,YAAY,EACV,qBAAqB,EACrB,cAAc,GACf,MAAM,6BAA6B,CAAC;AACrC,YAAY,EAAE,0BAA0B,EAAE,MAAM,mCAAmC,CAAC;AACpF,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,QAAQ,EACR,UAAU,EACV,QAAQ,EACR,eAAe,GAChB,MAAM,cAAc,CAAC;AACtB,YAAY,EACV,YAAY,EACZ,UAAU,EACV,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,YAAY,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACxD,YAAY,EACV,gBAAgB,EAChB,eAAe,EACf,eAAe,EACf,SAAS,GACV,MAAM,eAAe,CAAC;AACvB,YAAY,EACV,gBAAgB,EAChB,gBAAgB,EAChB,uBAAuB,EACvB,mBAAmB,GACpB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EACV,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,yBAAyB,EACzB,YAAY,GACb,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,QAAQ,EACR,mBAAmB,EACnB,YAAY,EACZ,aAAa,GACd,MAAM,cAAc,CAAC;AACtB,YAAY,EACV,aAAa,EACb,eAAe,EACf,iBAAiB,EACjB,aAAa,EACb,aAAa,EACb,aAAa,EACb,QAAQ,GACT,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,YAAY,EACV,eAAe,EACf,qBAAqB,EACrB,SAAS,EACT,cAAc,EACd,YAAY,EACZ,WAAW,EACX,IAAI,EACJ,iBAAiB,EACjB,UAAU,EACV,YAAY,EACZ,oBAAoB,EACpB,YAAY,EACZ,iBAAiB,EACjB,kBAAkB,GACnB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC/D,YAAY,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EACL,KAAK,kBAAkB,EACvB,mBAAmB,GACpB,MAAM,yBAAyB,CAAC;AACjC,YAAY,EACV,cAAc,EACd,YAAY,EACZ,YAAY,EACZ,aAAa,EACb,cAAc,EACd,WAAW,EACX,aAAa,EACb,aAAa,GACd,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC/C,YAAY,EACV,qBAAqB,EACrB,uBAAuB,EACvB,qBAAqB,EACrB,eAAe,EACf,WAAW,EACX,YAAY,EACZ,aAAa,GACd,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpD,YAAY,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7D,YAAY,EACV,MAAM,EACN,aAAa,EACb,UAAU,EACV,cAAc,GACf,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,cAAc,EACd,YAAY,EACZ,eAAe,EACf,gBAAgB,EAChB,YAAY,EACZ,cAAc,EACd,iBAAiB,EACjB,yBAAyB,EACzB,gBAAgB,EAChB,YAAY,EACZ,oBAAoB,EACpB,iBAAiB,EACjB,cAAc,EACd,QAAQ,EACR,iBAAiB,EACjB,eAAe,EACf,WAAW,EACX,yBAAyB,EACzB,cAAc,GACf,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,eAAe,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AAC3E,YAAY,EACV,YAAY,EACZ,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EACV,UAAU,EACV,oBAAoB,EACpB,WAAW,EACX,qBAAqB,EACrB,iBAAiB,EACjB,QAAQ,EACR,eAAe,EACf,kBAAkB,EAClB,mBAAmB,EACnB,cAAc,GACf,MAAM,6BAA6B,CAAC;AACrC,YAAY,EACV,MAAM,EACN,QAAQ,EACR,aAAa,EACb,cAAc,EACd,YAAY,EACZ,WAAW,GACZ,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,YAAY,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,YAAY,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,YAAY,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AACzE,YAAY,EACV,SAAS,EACT,cAAc,EACd,eAAe,EACf,oBAAoB,EACpB,gBAAgB,GACjB,MAAM,eAAe,CAAC;AACvB,YAAY,EACV,uBAAuB,EACvB,sBAAsB,EACtB,qBAAqB,GACtB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AACjE,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EACL,KAAK,iBAAiB,EACtB,gBAAgB,EAChB,KAAK,eAAe,EACpB,KAAK,gBAAgB,EACrB,UAAU,GACX,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,KAAK,oBAAoB,EACzB,qBAAqB,EACrB,KAAK,iBAAiB,GACvB,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EACV,wBAAwB,EACxB,SAAS,EACT,yBAAyB,EACzB,mBAAmB,EACnB,2BAA2B,EAC3B,8BAA8B,EAC9B,SAAS,GACV,MAAM,iCAAiC,CAAC;AACzC,OAAO,EACL,oBAAoB,EACpB,KAAK,cAAc,EACnB,KAAK,cAAc,EACnB,kBAAkB,GACnB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAE,sBAAsB,EAAE,MAAM,8BAA8B,CAAC;AAC3E,YAAY,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC3E,YAAY,EACV,WAAW,EACX,WAAW,EACX,YAAY,EACZ,eAAe,EACf,WAAW,EACX,cAAc,EACd,OAAO,EACP,WAAW,EACX,gBAAgB,EAChB,QAAQ,EACR,aAAa,EACb,eAAe,EACf,OAAO,EACP,aAAa,EACb,eAAe,EACf,WAAW,EACX,SAAS,EACT,MAAM,EACN,iBAAiB,EACjB,cAAc,EACd,SAAS,EACT,aAAa,EACb,WAAW,EACX,cAAc,EACd,aAAa,EACb,YAAY,GACb,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,cAAc,EACd,eAAe,EACf,gBAAgB,EAChB,QAAQ,EACR,qBAAqB,EACrB,QAAQ,EACR,QAAQ,EACR,UAAU,EACV,eAAe,GAChB,MAAM,YAAY,CAAC;AACpB,YAAY,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,YAAY,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC"}
|
package/dist/interfaces/index.js
CHANGED
|
@@ -5,6 +5,7 @@ export { isReadinessReporter, } from './readiness-reporter.js';
|
|
|
5
5
|
export { CatalogCasError, SkillsIncompatibleError } from './skills-rag.js';
|
|
6
6
|
export { TokenLedger } from './stepper.js';
|
|
7
7
|
export { renderTaskSpec } from './task-spec.js';
|
|
8
|
+
export { ReportThrottling, WaitAsTold, } from './throttle-strategy.js';
|
|
8
9
|
export { isToolCatalogReporter, } from './tool-catalog.js';
|
|
9
10
|
export { defaultToolRecordKey, toolNameFromRecord, } from './tool-record-key.js';
|
|
10
11
|
export { AssemblerError, ClassifierError, isMcpUnavailable, LlmError, MCP_UNAVAILABLE_CODES, McpError, RagError, SkillError, SmartAgentError, } from './types.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/interfaces/index.ts"],"names":[],"mappings":"AACA,OAAO,EAGL,iBAAiB,GAGlB,MAAM,sBAAsB,CAAC;AAiD9B,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAiG9D,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAE/D,OAAO,EAEL,mBAAmB,GACpB,MAAM,yBAAyB,CAAC;AAkDjC,OAAO,EAAE,eAAe,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AA0B3E,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAiB3C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAEL,qBAAqB,GAEtB,MAAM,mBAAmB,CAAC;AAU3B,OAAO,EACL,oBAAoB,EAGpB,kBAAkB,GACnB,MAAM,sBAAsB,CAAC;AA+B9B,OAAO,EACL,cAAc,EACd,eAAe,EACf,gBAAgB,EAChB,QAAQ,EACR,qBAAqB,EACrB,QAAQ,EACR,QAAQ,EACR,UAAU,EACV,eAAe,GAChB,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhD,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/interfaces/index.ts"],"names":[],"mappings":"AACA,OAAO,EAGL,iBAAiB,GAGlB,MAAM,sBAAsB,CAAC;AAiD9B,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAiG9D,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAE/D,OAAO,EAEL,mBAAmB,GACpB,MAAM,yBAAyB,CAAC;AAkDjC,OAAO,EAAE,eAAe,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AA0B3E,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAiB3C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAEL,gBAAgB,EAGhB,UAAU,GACX,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,qBAAqB,GAEtB,MAAM,mBAAmB,CAAC;AAU3B,OAAO,EACL,oBAAoB,EAGpB,kBAAkB,GACnB,MAAM,sBAAsB,CAAC;AA+B9B,OAAO,EACL,cAAc,EACd,eAAe,EACf,gBAAgB,EAChB,QAAQ,EACR,qBAAqB,EACrB,QAAQ,EACR,QAAQ,EACR,UAAU,EACV,eAAe,GAChB,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhD,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC"}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What a provider does when a server throttles it.
|
|
3
|
+
*
|
|
4
|
+
* Throttling, and specifically the SERVER'S. Client-side throttling is spacing
|
|
5
|
+
* calls out on our own initiative, and this library does none of it: it sets no
|
|
6
|
+
* interval, computes none, and has no number of its own to set one with. The
|
|
7
|
+
* server names a delay — `Retry-After` is either seconds or an HTTP-date, and
|
|
8
|
+
* both reduce to a delay from now — and we either observe it or report it.
|
|
9
|
+
* Where the server names nothing, there is no throttling to observe.
|
|
10
|
+
*
|
|
11
|
+
* The library establishes facts and decides nothing. This is a 429. The server
|
|
12
|
+
* named this interval, or named none. The quota is shut until then. What to do
|
|
13
|
+
* about it — wait, give up, count attempts, give up after the third — belongs
|
|
14
|
+
* entirely to the consumer, because the consumer is the only one who knows who
|
|
15
|
+
* is waiting at the other end.
|
|
16
|
+
*
|
|
17
|
+
* So `whenThrottled` is not a policy object with the library's numbers in it.
|
|
18
|
+
* It is the strategy itself. Anything a strategy wants to bound, it bounds on
|
|
19
|
+
* its own terms: a `maxAttempts` sitting outside would silently overrule a
|
|
20
|
+
* strategy that had already decided to keep going.
|
|
21
|
+
*
|
|
22
|
+
* Two ship, and neither invents a duration:
|
|
23
|
+
*
|
|
24
|
+
* - `ReportThrottling` — never waits. Returns the failure with what the server
|
|
25
|
+
* said, and the caller decides. This is what happens when nothing is set.
|
|
26
|
+
* - `WaitAsTold` — waits exactly the interval the server named, and reports
|
|
27
|
+
* when it named none, rather than guessing at milliseconds.
|
|
28
|
+
*/
|
|
29
|
+
/** What the strategy is told. All of it observed, none of it assumed. */
|
|
30
|
+
export interface ThrottleContext {
|
|
31
|
+
/**
|
|
32
|
+
* Where this came from, which changes what the interval means.
|
|
33
|
+
*
|
|
34
|
+
* `response` — a server just refused the call. `retryAfterSeconds` is what it
|
|
35
|
+
* said, and absent means it said nothing.
|
|
36
|
+
*
|
|
37
|
+
* `gate` — no request was sent, because the quota was already recorded as
|
|
38
|
+
* shut. `retryAfterSeconds` is what is left of OUR record, not a fresh
|
|
39
|
+
* statement from the server. A strategy that treats the two alike will be
|
|
40
|
+
* wrong about at least one of them.
|
|
41
|
+
*/
|
|
42
|
+
source: 'response' | 'gate';
|
|
43
|
+
/** 1-based. The attempt just refused, or 0 for a `gate` decision. */
|
|
44
|
+
attempt: number;
|
|
45
|
+
/** The interval, read as `source` says. */
|
|
46
|
+
retryAfterSeconds?: number;
|
|
47
|
+
/** How long this call has already spent waiting. */
|
|
48
|
+
waitedMs: number;
|
|
49
|
+
}
|
|
50
|
+
export interface ThrottleDecision {
|
|
51
|
+
/**
|
|
52
|
+
* How long this quota is shut, as far as we can tell.
|
|
53
|
+
*
|
|
54
|
+
* Recorded whether or not THIS caller waits: it is what the server said, and
|
|
55
|
+
* the next caller deserves to know it even if this one is leaving.
|
|
56
|
+
*/
|
|
57
|
+
waitMs: number;
|
|
58
|
+
/** Whether this caller sits it out and tries again. */
|
|
59
|
+
retry: boolean;
|
|
60
|
+
/** When not retrying, why — free text from a consumer's own strategy. */
|
|
61
|
+
reason?: string;
|
|
62
|
+
}
|
|
63
|
+
export interface IThrottleStrategy {
|
|
64
|
+
readonly name: string;
|
|
65
|
+
decide(ctx: ThrottleContext): ThrottleDecision;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Never wait. What happens when nothing is configured.
|
|
69
|
+
*
|
|
70
|
+
* The failure goes up carrying what the server said, and the caller — who knows
|
|
71
|
+
* whether anyone is waiting on the other end — decides whether to sit it out.
|
|
72
|
+
* The quota is still recorded as shut, so the next call does not spend a
|
|
73
|
+
* request discovering it.
|
|
74
|
+
*/
|
|
75
|
+
export declare class ReportThrottling implements IThrottleStrategy {
|
|
76
|
+
readonly name = "report";
|
|
77
|
+
decide({ retryAfterSeconds }: ThrottleContext): ThrottleDecision;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Wait exactly as long as the server asked, and only then.
|
|
81
|
+
*
|
|
82
|
+
* For a caller with nobody waiting on it — a batch job, a CLI — that would
|
|
83
|
+
* rather have the answer late than not at all.
|
|
84
|
+
*
|
|
85
|
+
* When the server names no interval this reports instead of guessing. A missing
|
|
86
|
+
* `Retry-After` is itself a signal: SAP AI Core documents the header, so its
|
|
87
|
+
* absence says something is not as expected, and a number invented here would
|
|
88
|
+
* be an estimate of a system we cannot see.
|
|
89
|
+
*
|
|
90
|
+
* `maxAttempts` is the strategy's own, because choosing this strategy is the
|
|
91
|
+
* consumer's act and so is bounding it. Omit it and the strategy keeps
|
|
92
|
+
* returning for as long as the server keeps naming an interval; bound that with
|
|
93
|
+
* an `AbortSignal`, which is a deadline from the caller's own clock.
|
|
94
|
+
*/
|
|
95
|
+
export declare class WaitAsTold implements IThrottleStrategy {
|
|
96
|
+
readonly name = "wait-as-told";
|
|
97
|
+
private readonly maxAttempts;
|
|
98
|
+
constructor(options?: {
|
|
99
|
+
maxAttempts?: number;
|
|
100
|
+
});
|
|
101
|
+
decide({ attempt, retryAfterSeconds }: ThrottleContext): ThrottleDecision;
|
|
102
|
+
}
|
|
103
|
+
//# sourceMappingURL=throttle-strategy.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"throttle-strategy.d.ts","sourceRoot":"","sources":["../../src/interfaces/throttle-strategy.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,yEAAyE;AACzE,MAAM,WAAW,eAAe;IAC9B;;;;;;;;;;OAUG;IACH,MAAM,EAAE,UAAU,GAAG,MAAM,CAAC;IAC5B,qEAAqE;IACrE,OAAO,EAAE,MAAM,CAAC;IAChB,2CAA2C;IAC3C,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,oDAAoD;IACpD,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,gBAAgB;IAC/B;;;;;OAKG;IACH,MAAM,EAAE,MAAM,CAAC;IACf,uDAAuD;IACvD,KAAK,EAAE,OAAO,CAAC;IACf,yEAAyE;IACzE,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,GAAG,EAAE,eAAe,GAAG,gBAAgB,CAAC;CAChD;AASD;;;;;;;GAOG;AACH,qBAAa,gBAAiB,YAAW,iBAAiB;IACxD,QAAQ,CAAC,IAAI,YAAY;IAEzB,MAAM,CAAC,EAAE,iBAAiB,EAAE,EAAE,eAAe,GAAG,gBAAgB;CAOjE;AAED;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,UAAW,YAAW,iBAAiB;IAClD,QAAQ,CAAC,IAAI,kBAAkB;IAC/B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;gBAEzB,OAAO,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO;IAIlD,MAAM,CAAC,EAAE,OAAO,EAAE,iBAAiB,EAAE,EAAE,eAAe,GAAG,gBAAgB;CAU1E"}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What a provider does when a server throttles it.
|
|
3
|
+
*
|
|
4
|
+
* Throttling, and specifically the SERVER'S. Client-side throttling is spacing
|
|
5
|
+
* calls out on our own initiative, and this library does none of it: it sets no
|
|
6
|
+
* interval, computes none, and has no number of its own to set one with. The
|
|
7
|
+
* server names a delay — `Retry-After` is either seconds or an HTTP-date, and
|
|
8
|
+
* both reduce to a delay from now — and we either observe it or report it.
|
|
9
|
+
* Where the server names nothing, there is no throttling to observe.
|
|
10
|
+
*
|
|
11
|
+
* The library establishes facts and decides nothing. This is a 429. The server
|
|
12
|
+
* named this interval, or named none. The quota is shut until then. What to do
|
|
13
|
+
* about it — wait, give up, count attempts, give up after the third — belongs
|
|
14
|
+
* entirely to the consumer, because the consumer is the only one who knows who
|
|
15
|
+
* is waiting at the other end.
|
|
16
|
+
*
|
|
17
|
+
* So `whenThrottled` is not a policy object with the library's numbers in it.
|
|
18
|
+
* It is the strategy itself. Anything a strategy wants to bound, it bounds on
|
|
19
|
+
* its own terms: a `maxAttempts` sitting outside would silently overrule a
|
|
20
|
+
* strategy that had already decided to keep going.
|
|
21
|
+
*
|
|
22
|
+
* Two ship, and neither invents a duration:
|
|
23
|
+
*
|
|
24
|
+
* - `ReportThrottling` — never waits. Returns the failure with what the server
|
|
25
|
+
* said, and the caller decides. This is what happens when nothing is set.
|
|
26
|
+
* - `WaitAsTold` — waits exactly the interval the server named, and reports
|
|
27
|
+
* when it named none, rather than guessing at milliseconds.
|
|
28
|
+
*/
|
|
29
|
+
/** Seconds the server named, in milliseconds, or 0 when it named none. */
|
|
30
|
+
function servedWaitMs(retryAfterSeconds) {
|
|
31
|
+
return retryAfterSeconds !== undefined && Number.isFinite(retryAfterSeconds)
|
|
32
|
+
? retryAfterSeconds * 1000
|
|
33
|
+
: 0;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Never wait. What happens when nothing is configured.
|
|
37
|
+
*
|
|
38
|
+
* The failure goes up carrying what the server said, and the caller — who knows
|
|
39
|
+
* whether anyone is waiting on the other end — decides whether to sit it out.
|
|
40
|
+
* The quota is still recorded as shut, so the next call does not spend a
|
|
41
|
+
* request discovering it.
|
|
42
|
+
*/
|
|
43
|
+
export class ReportThrottling {
|
|
44
|
+
name = 'report';
|
|
45
|
+
decide({ retryAfterSeconds }) {
|
|
46
|
+
return {
|
|
47
|
+
waitMs: servedWaitMs(retryAfterSeconds),
|
|
48
|
+
retry: false,
|
|
49
|
+
reason: 'reported',
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Wait exactly as long as the server asked, and only then.
|
|
55
|
+
*
|
|
56
|
+
* For a caller with nobody waiting on it — a batch job, a CLI — that would
|
|
57
|
+
* rather have the answer late than not at all.
|
|
58
|
+
*
|
|
59
|
+
* When the server names no interval this reports instead of guessing. A missing
|
|
60
|
+
* `Retry-After` is itself a signal: SAP AI Core documents the header, so its
|
|
61
|
+
* absence says something is not as expected, and a number invented here would
|
|
62
|
+
* be an estimate of a system we cannot see.
|
|
63
|
+
*
|
|
64
|
+
* `maxAttempts` is the strategy's own, because choosing this strategy is the
|
|
65
|
+
* consumer's act and so is bounding it. Omit it and the strategy keeps
|
|
66
|
+
* returning for as long as the server keeps naming an interval; bound that with
|
|
67
|
+
* an `AbortSignal`, which is a deadline from the caller's own clock.
|
|
68
|
+
*/
|
|
69
|
+
export class WaitAsTold {
|
|
70
|
+
name = 'wait-as-told';
|
|
71
|
+
maxAttempts;
|
|
72
|
+
constructor(options = {}) {
|
|
73
|
+
this.maxAttempts = options.maxAttempts ?? Number.POSITIVE_INFINITY;
|
|
74
|
+
}
|
|
75
|
+
decide({ attempt, retryAfterSeconds }) {
|
|
76
|
+
const waitMs = servedWaitMs(retryAfterSeconds);
|
|
77
|
+
if (waitMs <= 0) {
|
|
78
|
+
return { waitMs: 0, retry: false, reason: 'no-interval' };
|
|
79
|
+
}
|
|
80
|
+
if (attempt >= this.maxAttempts) {
|
|
81
|
+
return { waitMs, retry: false, reason: 'attempts' };
|
|
82
|
+
}
|
|
83
|
+
return { waitMs, retry: true };
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
//# sourceMappingURL=throttle-strategy.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"throttle-strategy.js","sourceRoot":"","sources":["../../src/interfaces/throttle-strategy.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AA2CH,0EAA0E;AAC1E,SAAS,YAAY,CAAC,iBAAqC;IACzD,OAAO,iBAAiB,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAC1E,CAAC,CAAC,iBAAiB,GAAG,IAAI;QAC1B,CAAC,CAAC,CAAC,CAAC;AACR,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,OAAO,gBAAgB;IAClB,IAAI,GAAG,QAAQ,CAAC;IAEzB,MAAM,CAAC,EAAE,iBAAiB,EAAmB;QAC3C,OAAO;YACL,MAAM,EAAE,YAAY,CAAC,iBAAiB,CAAC;YACvC,KAAK,EAAE,KAAK;YACZ,MAAM,EAAE,UAAU;SACnB,CAAC;IACJ,CAAC;CACF;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,OAAO,UAAU;IACZ,IAAI,GAAG,cAAc,CAAC;IACd,WAAW,CAAS;IAErC,YAAY,UAAoC,EAAE;QAChD,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,MAAM,CAAC,iBAAiB,CAAC;IACrE,CAAC;IAED,MAAM,CAAC,EAAE,OAAO,EAAE,iBAAiB,EAAmB;QACpD,MAAM,MAAM,GAAG,YAAY,CAAC,iBAAiB,CAAC,CAAC;QAC/C,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;YAChB,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;QAC5D,CAAC;QACD,IAAI,OAAO,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YAChC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;QACtD,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IACjC,CAAC;CACF"}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Base interface for LLM providers
|
|
3
3
|
*/
|
|
4
4
|
import type { IModelInfo, LLMCallOptions, LLMProviderConfig, LLMResponse, Message } from '@mcp-abap-adt/llm-agent';
|
|
5
|
-
import { type
|
|
5
|
+
import { type ThrottleRetryOptions } from './throttle.js';
|
|
6
6
|
export interface LLMProvider {
|
|
7
7
|
/**
|
|
8
8
|
* Send a chat message and get response
|
|
@@ -43,7 +43,7 @@ export declare abstract class BaseLLMProvider<C extends LLMProviderConfig = LLMP
|
|
|
43
43
|
* the account's secret. Override to add a provider's own account fields
|
|
44
44
|
* (organization, project, resource group) — never the raw secret.
|
|
45
45
|
*/
|
|
46
|
-
protected
|
|
46
|
+
protected quotaScope(): string;
|
|
47
47
|
/**
|
|
48
48
|
* Reduce spellings of one endpoint to one key.
|
|
49
49
|
*
|
|
@@ -74,14 +74,22 @@ export declare abstract class BaseLLMProvider<C extends LLMProviderConfig = LLMP
|
|
|
74
74
|
* model's 429 would pause another's, and two overrides would share a gate
|
|
75
75
|
* neither of them belongs to.
|
|
76
76
|
*/
|
|
77
|
-
protected
|
|
77
|
+
protected quotaKey(model?: string): string;
|
|
78
78
|
/** Is this "too many requests" rather than a real failure? */
|
|
79
|
-
protected
|
|
79
|
+
protected isThrottled(error: unknown): boolean;
|
|
80
80
|
/**
|
|
81
|
-
* How long the server asked us to wait.
|
|
81
|
+
* How long the server asked us to wait, or undefined if it did not say.
|
|
82
82
|
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
83
|
+
* RFC 9110 allows two forms, `delay-seconds` and an HTTP-date, and they are
|
|
84
|
+
* told apart by the fact that a valid date never parses as a number. SAP AI
|
|
85
|
+
* Core documents seconds ("Time in seconds to wait before retrying"); the
|
|
86
|
+
* date form is read because the spec permits it, not because anyone sends it.
|
|
87
|
+
*
|
|
88
|
+
* An empty or blank header is "did not say", not "zero". `Number('')` is 0,
|
|
89
|
+
* so reading it arithmetically would have us report an interval the server
|
|
90
|
+
* never named — a lie in the one place it costs most, since the absence is
|
|
91
|
+
* itself the signal. Anthropic returns a 429 with no `Retry-After` when a
|
|
92
|
+
* spend cap is reached, and that one does not clear by waiting at all.
|
|
85
93
|
*/
|
|
86
94
|
protected retryAfterSeconds(error: unknown): number | undefined;
|
|
87
95
|
/**
|
|
@@ -90,13 +98,13 @@ export declare abstract class BaseLLMProvider<C extends LLMProviderConfig = LLMP
|
|
|
90
98
|
* Providers rethrow their transport error as a readable one. Call this on the
|
|
91
99
|
* way out so the consumer still reads a fact, not a substring.
|
|
92
100
|
*/
|
|
93
|
-
protected
|
|
101
|
+
protected preserveThrottled<E extends Error>(original: unknown, wrapped: E): E;
|
|
94
102
|
/** Wrap one provider call in the shared policy. */
|
|
95
|
-
protected
|
|
103
|
+
protected withThrottleRetry<T>(fn: () => Promise<T>, extra?: {
|
|
96
104
|
/** The model this call actually uses, when it overrides the configured one. */
|
|
97
105
|
model?: string;
|
|
98
106
|
signal?: AbortSignal;
|
|
99
|
-
onRetry?:
|
|
107
|
+
onRetry?: ThrottleRetryOptions['onRetry'];
|
|
100
108
|
}): Promise<T>;
|
|
101
109
|
}
|
|
102
110
|
//# sourceMappingURL=base-llm-provider.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base-llm-provider.d.ts","sourceRoot":"","sources":["../../src/llm/base-llm-provider.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EACV,UAAU,EACV,cAAc,EACd,iBAAiB,EACjB,WAAW,EACX,OAAO,EACR,MAAM,yBAAyB,CAAC;AACjC,OAAO,
|
|
1
|
+
{"version":3,"file":"base-llm-provider.d.ts","sourceRoot":"","sources":["../../src/llm/base-llm-provider.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EACV,UAAU,EACV,cAAc,EACd,iBAAiB,EACjB,WAAW,EACX,OAAO,EACR,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAGL,KAAK,oBAAoB,EAC1B,MAAM,eAAe,CAAC;AAEvB,MAAM,WAAW,WAAW;IAC1B;;OAEG;IACH,IAAI,CACF,QAAQ,EAAE,OAAO,EAAE,EACnB,KAAK,CAAC,EAAE,OAAO,EAAE,EACjB,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,WAAW,CAAC,CAAC;IAExB;;OAEG;IACH,UAAU,CACR,QAAQ,EAAE,OAAO,EAAE,EACnB,KAAK,CAAC,EAAE,OAAO,EAAE,EACjB,OAAO,CAAC,EAAE,cAAc,GACvB,aAAa,CAAC,WAAW,CAAC,CAAC;IAE9B;;OAEG;IACH,SAAS,CAAC,IAAI,OAAO,CAAC,MAAM,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC;IAE/C;;OAEG;IACH,kBAAkB,CAAC,IAAI,OAAO,CAAC,MAAM,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC;CACzD;AAED,8BAAsB,eAAe,CACnC,CAAC,SAAS,iBAAiB,GAAG,iBAAiB,CAC/C,YAAW,WAAW;IAEtB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;gBAEP,MAAM,EAAE,CAAC;IAIrB,QAAQ,CAAC,IAAI,CACX,QAAQ,EAAE,OAAO,EAAE,EACnB,KAAK,CAAC,EAAE,OAAO,EAAE,EACjB,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,WAAW,CAAC;IAEvB,QAAQ,CAAC,UAAU,CACjB,QAAQ,EAAE,OAAO,EAAE,EACnB,KAAK,CAAC,EAAE,OAAO,EAAE,EACjB,OAAO,CAAC,EAAE,cAAc,GACvB,aAAa,CAAC,WAAW,CAAC;IAE7B;;OAEG;IACH,SAAS,CAAC,cAAc,IAAI,IAAI;IAchC;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,UAAU,IAAI,MAAM;IAO9B;;;;;;;;;;;OAWG;IACH,SAAS,CAAC,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAahD;;;;;;OAMG;IACH,SAAS,CAAC,aAAa,IAAI,MAAM;IAIjC,4EAA4E;IAC5E,SAAS,CAAC,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM;IAInE;;;;;;OAMG;IACH,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM;IAM1C,8DAA8D;IAC9D,SAAS,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO;IAI9C;;;;;;;;;;;;;OAaG;IACH,SAAS,CAAC,iBAAiB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS;IAiB/D;;;;;OAKG;IACH,SAAS,CAAC,iBAAiB,CAAC,CAAC,SAAS,KAAK,EACzC,QAAQ,EAAE,OAAO,EACjB,OAAO,EAAE,CAAC,GACT,CAAC;IAIJ,mDAAmD;IACnD,SAAS,CAAC,iBAAiB,CAAC,CAAC,EAC3B,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACpB,KAAK,CAAC,EAAE;QACN,+EAA+E;QAC/E,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,OAAO,CAAC,EAAE,oBAAoB,CAAC,SAAS,CAAC,CAAC;KAC3C,GACA,OAAO,CAAC,CAAC,CAAC;CAUd"}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Base interface for LLM providers
|
|
3
3
|
*/
|
|
4
4
|
import { createHash } from 'node:crypto';
|
|
5
|
-
import {
|
|
5
|
+
import { preserveThrottled, runWithThrottleRetry, } from './throttle.js';
|
|
6
6
|
export class BaseLLMProvider {
|
|
7
7
|
config;
|
|
8
8
|
constructor(config) {
|
|
@@ -36,7 +36,7 @@ export class BaseLLMProvider {
|
|
|
36
36
|
* the account's secret. Override to add a provider's own account fields
|
|
37
37
|
* (organization, project, resource group) — never the raw secret.
|
|
38
38
|
*/
|
|
39
|
-
|
|
39
|
+
quotaScope() {
|
|
40
40
|
return [
|
|
41
41
|
this.canonicalEndpoint(this.quotaEndpoint()),
|
|
42
42
|
this.credentialFingerprint(this.config.apiKey),
|
|
@@ -89,27 +89,39 @@ export class BaseLLMProvider {
|
|
|
89
89
|
* model's 429 would pause another's, and two overrides would share a gate
|
|
90
90
|
* neither of them belongs to.
|
|
91
91
|
*/
|
|
92
|
-
|
|
93
|
-
return `${this.constructor.name}:${this.
|
|
92
|
+
quotaKey(model) {
|
|
93
|
+
return `${this.constructor.name}:${this.quotaScope()}:${model ?? this.config.model ?? 'default'}`;
|
|
94
94
|
}
|
|
95
95
|
/** Is this "too many requests" rather than a real failure? */
|
|
96
|
-
|
|
96
|
+
isThrottled(error) {
|
|
97
97
|
return httpStatusOf(error) === 429;
|
|
98
98
|
}
|
|
99
99
|
/**
|
|
100
|
-
* How long the server asked us to wait.
|
|
100
|
+
* How long the server asked us to wait, or undefined if it did not say.
|
|
101
101
|
*
|
|
102
|
-
*
|
|
103
|
-
*
|
|
102
|
+
* RFC 9110 allows two forms, `delay-seconds` and an HTTP-date, and they are
|
|
103
|
+
* told apart by the fact that a valid date never parses as a number. SAP AI
|
|
104
|
+
* Core documents seconds ("Time in seconds to wait before retrying"); the
|
|
105
|
+
* date form is read because the spec permits it, not because anyone sends it.
|
|
106
|
+
*
|
|
107
|
+
* An empty or blank header is "did not say", not "zero". `Number('')` is 0,
|
|
108
|
+
* so reading it arithmetically would have us report an interval the server
|
|
109
|
+
* never named — a lie in the one place it costs most, since the absence is
|
|
110
|
+
* itself the signal. Anthropic returns a 429 with no `Retry-After` when a
|
|
111
|
+
* spend cap is reached, and that one does not clear by waiting at all.
|
|
104
112
|
*/
|
|
105
113
|
retryAfterSeconds(error) {
|
|
106
|
-
const raw = headerOf(error, 'retry-after');
|
|
107
|
-
if (raw === undefined)
|
|
114
|
+
const raw = headerOf(error, 'retry-after')?.trim();
|
|
115
|
+
if (raw === undefined || raw === '')
|
|
108
116
|
return undefined;
|
|
117
|
+
// Numeric first, and once it is numeric it is never reconsidered as a date.
|
|
118
|
+
// Node's fallback date parser is lenient enough to read '-5' as the year
|
|
119
|
+
// 2001, which would turn a malformed header into "retry immediately".
|
|
109
120
|
const seconds = Number(raw);
|
|
110
|
-
if (Number.
|
|
111
|
-
return seconds;
|
|
112
|
-
|
|
121
|
+
if (!Number.isNaN(seconds)) {
|
|
122
|
+
return Number.isFinite(seconds) && seconds >= 0 ? seconds : undefined;
|
|
123
|
+
}
|
|
124
|
+
const when = Date.parse(raw);
|
|
113
125
|
if (!Number.isNaN(when))
|
|
114
126
|
return Math.max(0, (when - Date.now()) / 1000);
|
|
115
127
|
return undefined;
|
|
@@ -120,15 +132,15 @@ export class BaseLLMProvider {
|
|
|
120
132
|
* Providers rethrow their transport error as a readable one. Call this on the
|
|
121
133
|
* way out so the consumer still reads a fact, not a substring.
|
|
122
134
|
*/
|
|
123
|
-
|
|
124
|
-
return
|
|
135
|
+
preserveThrottled(original, wrapped) {
|
|
136
|
+
return preserveThrottled(original, wrapped);
|
|
125
137
|
}
|
|
126
138
|
/** Wrap one provider call in the shared policy. */
|
|
127
|
-
|
|
128
|
-
return
|
|
129
|
-
key: this.
|
|
130
|
-
|
|
131
|
-
|
|
139
|
+
withThrottleRetry(fn, extra) {
|
|
140
|
+
return runWithThrottleRetry(fn, {
|
|
141
|
+
key: this.quotaKey(extra?.model),
|
|
142
|
+
strategy: this.config.whenThrottled,
|
|
143
|
+
isThrottled: (e) => this.isThrottled(e),
|
|
132
144
|
retryAfterSeconds: (e) => this.retryAfterSeconds(e),
|
|
133
145
|
signal: extra?.signal,
|
|
134
146
|
onRetry: extra?.onRetry,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base-llm-provider.js","sourceRoot":"","sources":["../../src/llm/base-llm-provider.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAQzC,OAAO,EACL,iBAAiB,
|
|
1
|
+
{"version":3,"file":"base-llm-provider.js","sourceRoot":"","sources":["../../src/llm/base-llm-provider.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAQzC,OAAO,EACL,iBAAiB,EACjB,oBAAoB,GAErB,MAAM,eAAe,CAAC;AAgCvB,MAAM,OAAgB,eAAe;IAI1B,MAAM,CAAI;IAEnB,YAAY,MAAS;QACnB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAcD;;OAEG;IACO,cAAc;QACtB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IAED,2EAA2E;IAC3E,EAAE;IACF,2EAA2E;IAC3E,2EAA2E;IAC3E,6EAA6E;IAC7E,4EAA4E;IAC5E,uEAAuE;IAEvE;;;;;;;;;;;;OAYG;IACO,UAAU;QAClB,OAAO;YACL,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YAC5C,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;SAC/C,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACd,CAAC;IAED;;;;;;;;;;;OAWG;IACO,iBAAiB,CAAC,GAAW;QACrC,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QAC3B,IAAI,CAAC,OAAO;YAAE,OAAO,SAAS,CAAC;QAC/B,IAAI,CAAC;YACH,kEAAkE;YAClE,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;YAC7B,OAAO,GAAG,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;QACzE,CAAC;QAAC,MAAM,CAAC;YACP,iEAAiE;YACjE,OAAO,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACO,aAAa;QACrB,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,SAAS,CAAC;IAC1C,CAAC;IAED,4EAA4E;IAClE,qBAAqB,CAAC,MAA0B;QACxD,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC;IAC7B,CAAC;IAED;;;;;;OAMG;IACO,QAAQ,CAAC,KAAc;QAC/B,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,EAAE,IAClD,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,SAChC,EAAE,CAAC;IACL,CAAC;IAED,8DAA8D;IACpD,WAAW,CAAC,KAAc;QAClC,OAAO,YAAY,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC;IACrC,CAAC;IAED;;;;;;;;;;;;;OAaG;IACO,iBAAiB,CAAC,KAAc;QACxC,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,EAAE,IAAI,EAAE,CAAC;QACnD,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,EAAE;YAAE,OAAO,SAAS,CAAC;QAEtD,4EAA4E;QAC5E,yEAAyE;QACzE,sEAAsE;QACtE,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3B,OAAO,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;QACxE,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QACxE,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;;;;OAKG;IACO,iBAAiB,CACzB,QAAiB,EACjB,OAAU;QAEV,OAAO,iBAAiB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC9C,CAAC;IAED,mDAAmD;IACzC,iBAAiB,CACzB,EAAoB,EACpB,KAKC;QAED,OAAO,oBAAoB,CAAC,EAAE,EAAE;YAC9B,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC;YAChC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa;YACnC,WAAW,EAAE,CAAC,CAAU,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;YAChD,iBAAiB,EAAE,CAAC,CAAU,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;YAC5D,MAAM,EAAE,KAAK,EAAE,MAAM;YACrB,OAAO,EAAE,KAAK,EAAE,OAAO;SACxB,CAAC,CAAC;IACL,CAAC;CACF;AAED;;;;;;GAMG;AACH,SAAS,WAAW,CAAC,MAA0B;IAC7C,IAAI,CAAC,MAAM;QAAE,OAAO,WAAW,CAAC;IAChC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACxE,CAAC;AAED,iFAAiF;AACjF,SAAS,YAAY,CAAC,KAAc;IAClC,MAAM,CAAC,GAAG,KAIT,CAAC;IACF,8EAA8E;IAC9E,KAAK,MAAM,SAAS,IAAI,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,UAAU,CAAC,EAAE,CAAC;QACxE,IAAI,OAAO,SAAS,KAAK,QAAQ;YAAE,OAAO,SAAS,CAAC;IACtD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,2EAA2E;AAC3E,SAAS,QAAQ,CAAC,KAAc,EAAE,IAAY;IAC5C,MAAM,OAAO,GAAI,KAA8C,EAAE,QAAQ;QACvE,EAAE,OAAO,CAAC;IACZ,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC9D,yDAAyD;IACzD,MAAM,GAAG,GAAI,OAA4C,CAAC,GAAG,CAAC;IAC9D,IAAI,OAAO,GAAG,KAAK,UAAU,EAAE,CAAC;QAC9B,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAClC,OAAO,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC/D,CAAC;IACD,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAkC,CAAC,EAAE,CAAC;QACxE,IAAI,CAAC,CAAC,WAAW,EAAE,KAAK,IAAI;YAC1B,OAAO,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACjE,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared handling of server-governed throttling, for every LLM provider.
|
|
3
|
+
*
|
|
4
|
+
* None of the providers handled HTTP 429 (issue #282). Each one let the status
|
|
5
|
+
* collapse into an error message, so consumers were reduced to matching the
|
|
6
|
+
* substring "429" in prose to decide whether to retry.
|
|
7
|
+
*
|
|
8
|
+
* The policy lives here rather than in a provider because the providers differ
|
|
9
|
+
* in SDK but not in the shape of the problem: a shared quota, a documented
|
|
10
|
+
* `Retry-After`, and a herd of concurrent callers that will all rediscover the
|
|
11
|
+
* same limit unless something holds them back together.
|
|
12
|
+
*
|
|
13
|
+
* The delay is always the server's. `Retry-After` may arrive as seconds or as
|
|
14
|
+
* an HTTP-date; both reduce to a delay from now, which is why "throttling" is
|
|
15
|
+
* the right word for it and why none of it is ours to invent.
|
|
16
|
+
*
|
|
17
|
+
* SAP AI Core states the rules this follows
|
|
18
|
+
* (help.sap.com, Rate Limit Management):
|
|
19
|
+
*
|
|
20
|
+
* Do not retry immediately after a 429. Use exponential backoff with jitter
|
|
21
|
+
* to prevent thundering-herd behavior. If the response includes a
|
|
22
|
+
* `Retry-After` header, use its value as the wait interval. Limit retries by
|
|
23
|
+
* using a maximum cap (for example, 5 retries or 60 seconds total wait).
|
|
24
|
+
*
|
|
25
|
+
* The same rules fit Anthropic and OpenAI, which return 429 with `Retry-After`
|
|
26
|
+
* as ordinary HTTP. Ollama is local and never rate-limited; it simply never
|
|
27
|
+
* enters this path.
|
|
28
|
+
*/
|
|
29
|
+
import { type IThrottleStrategy, type ThrottleDecision } from '../interfaces/throttle-strategy.js';
|
|
30
|
+
/** An error annotated by this module, so consumers read a fact instead of prose. */
|
|
31
|
+
export interface ThrottledError extends Error {
|
|
32
|
+
throttled: true;
|
|
33
|
+
/** What the server asked for, when it asked. */
|
|
34
|
+
retryAfterSeconds?: number;
|
|
35
|
+
/** How many attempts were spent before giving up. */
|
|
36
|
+
attempts: number;
|
|
37
|
+
/**
|
|
38
|
+
* Which cap ended it, when it ended here. `attempts` and `budget` are fixed
|
|
39
|
+
* by opposite settings, so a consumer that cannot tell them apart cannot act
|
|
40
|
+
* on either. `gate` means the pause was already longer than the budget and no
|
|
41
|
+
* request was sent at all.
|
|
42
|
+
*/
|
|
43
|
+
reason?: ThrottleDecision['reason'];
|
|
44
|
+
}
|
|
45
|
+
export declare function isThrottledError(e: unknown): e is ThrottledError;
|
|
46
|
+
/**
|
|
47
|
+
* One shared pause per quota.
|
|
48
|
+
*
|
|
49
|
+
* A rate limit belongs to the quota, not to the request that happened to
|
|
50
|
+
* discover it. When one call is told to wait, every other call against the same
|
|
51
|
+
* quota is already over the limit too — letting each find that out for itself
|
|
52
|
+
* means N more rejected requests and N more penalties, which is how a limit
|
|
53
|
+
* that should last one window lasts several.
|
|
54
|
+
*
|
|
55
|
+
* Keyed because limits are per model: one model being throttled says nothing
|
|
56
|
+
* about another.
|
|
57
|
+
*/
|
|
58
|
+
export declare class QuotaGate {
|
|
59
|
+
private notBefore;
|
|
60
|
+
/** Hold every caller until the named time. Never shortens an existing hold. */
|
|
61
|
+
penalise(ms: number, now?: number): void;
|
|
62
|
+
/** Milliseconds still to wait, or 0. */
|
|
63
|
+
remaining(now?: number): number;
|
|
64
|
+
clear(): void;
|
|
65
|
+
/**
|
|
66
|
+
* Wait out any active hold.
|
|
67
|
+
*
|
|
68
|
+
* Re-checked after every wake, because the hold is not fixed at the moment
|
|
69
|
+
* this caller started waiting: another caller's 429 extends it, and a sleeper
|
|
70
|
+
* timed against the old deadline would wake early and spend a request on a
|
|
71
|
+
* quota the server has since said is still closed.
|
|
72
|
+
*
|
|
73
|
+
* The wait is jittered per caller: without that, everyone released by one
|
|
74
|
+
* penalty wakes in the same millisecond and rebuilds the herd the penalty was
|
|
75
|
+
* meant to break up.
|
|
76
|
+
*/
|
|
77
|
+
waitUntilOpen(signal?: AbortSignal, maxWaitMs?: number): Promise<void>;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Keys are open-ended — a per-request model override mints a new one — so the
|
|
81
|
+
* registry is pruned rather than left to grow for the life of the process.
|
|
82
|
+
* Only entries that are both open and long idle are dropped: a held gate is
|
|
83
|
+
* doing its job, and a recently used one may still be someone's in-flight call.
|
|
84
|
+
*/
|
|
85
|
+
export declare const GATE_IDLE_TTL_MS: number;
|
|
86
|
+
/**
|
|
87
|
+
* How many quotas the registry keeps. Enforced by evicting gates that are both
|
|
88
|
+
* open and unheld, so a process with more than this many pauses or in-flight
|
|
89
|
+
* calls at once can exceed it — those gates are load-bearing. Evicting an idle
|
|
90
|
+
* one costs nothing: it carries no deadline, and the next call mints an
|
|
91
|
+
* equivalent.
|
|
92
|
+
*/
|
|
93
|
+
export declare const GATE_LIMIT = 500;
|
|
94
|
+
/** The gate for one quota. Shared across every provider instance in the process. */
|
|
95
|
+
export declare function gateFor(key: string): QuotaGate;
|
|
96
|
+
/**
|
|
97
|
+
* Hold a gate for the length of one call, and keep it in the registry meanwhile.
|
|
98
|
+
*
|
|
99
|
+
* Being open is not the same as being unused. A request is in flight for as
|
|
100
|
+
* long as it takes the server to answer, and until that answer arrives the gate
|
|
101
|
+
* it belongs to has no pause on it. Evicted in that window, the call would go on
|
|
102
|
+
* to penalise an object the registry no longer knows: the pause would be real
|
|
103
|
+
* and invisible, and the next caller would be handed a fresh open gate and sent
|
|
104
|
+
* straight into a quota that had just closed. Worse than no gate at all, since
|
|
105
|
+
* the penalty is paid and nothing is bought with it.
|
|
106
|
+
*
|
|
107
|
+
* Release in a `finally`. Releasing twice is harmless.
|
|
108
|
+
*/
|
|
109
|
+
export declare function leaseQuotaGate(key: string): {
|
|
110
|
+
gate: QuotaGate;
|
|
111
|
+
release: () => void;
|
|
112
|
+
};
|
|
113
|
+
/**
|
|
114
|
+
* Reclaim gates that are open and long idle. Exported so a long-lived host can
|
|
115
|
+
* run it on a timer, and so the reclaim can be tested at a chosen `now` without
|
|
116
|
+
* waiting ten minutes. It frees only what is certainly stale; the size bound is
|
|
117
|
+
* enforced separately, by `reclaimGates`.
|
|
118
|
+
*/
|
|
119
|
+
export declare function pruneQuotaGates(now?: number): void;
|
|
120
|
+
/** How many quotas are currently tracked. Diagnostics and tests. */
|
|
121
|
+
export declare function quotaGateCount(): number;
|
|
122
|
+
/** Test seam — drops every gate. */
|
|
123
|
+
export declare function resetQuotaGates(): void;
|
|
124
|
+
export interface ThrottleRetryOptions {
|
|
125
|
+
/** Identifies the quota being spent. Limits are per model, so include it. */
|
|
126
|
+
key: string;
|
|
127
|
+
/** What to do about a 429. Omit and nothing waits. */
|
|
128
|
+
strategy?: IThrottleStrategy;
|
|
129
|
+
/** Does this error mean "too many requests"? */
|
|
130
|
+
isThrottled: (error: unknown) => boolean;
|
|
131
|
+
/** What the server asked us to wait, in seconds, if it said. */
|
|
132
|
+
retryAfterSeconds?: (error: unknown) => number | undefined;
|
|
133
|
+
signal?: AbortSignal;
|
|
134
|
+
onRetry?: (info: {
|
|
135
|
+
attempt: number;
|
|
136
|
+
delayMs: number;
|
|
137
|
+
retryAfterSeconds?: number;
|
|
138
|
+
}) => void;
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* What happened, every time a server says 429.
|
|
142
|
+
*
|
|
143
|
+
* Reported for every provider and on every refusal, including the last one —
|
|
144
|
+
* the case an operator most needs to see was the one nothing was written for.
|
|
145
|
+
*
|
|
146
|
+
* The policy travels with the event because the numbers are unreadable without
|
|
147
|
+
* it: `attempt 3` is the end under `maxAttempts: 3` and the middle under 5. And
|
|
148
|
+
* because it answers the question a configuration crossing several layers
|
|
149
|
+
* otherwise leaves open, which is whether it arrived at all.
|
|
150
|
+
*/
|
|
151
|
+
export interface ThrottleEvent {
|
|
152
|
+
/** The quota: account, endpoint and model. Carries no credential. */
|
|
153
|
+
key: string;
|
|
154
|
+
/** The strategy that made the call. */
|
|
155
|
+
strategy: string;
|
|
156
|
+
/**
|
|
157
|
+
* Where this came from.
|
|
158
|
+
*
|
|
159
|
+
* `response` — a server refused a call we made. `retryAfterSeconds` is the
|
|
160
|
+
* server's own, and absent means it sent no header: that is the fact this
|
|
161
|
+
* event exists to establish, so it must not be confused with the other case.
|
|
162
|
+
*
|
|
163
|
+
* `gate` — no request was sent, the quota being already recorded as shut.
|
|
164
|
+
* The interval is what is left of our own record.
|
|
165
|
+
*/
|
|
166
|
+
source: 'response' | 'gate';
|
|
167
|
+
/** 1-based, the attempt that was just refused. 0 for a `gate` event. */
|
|
168
|
+
attempt: number;
|
|
169
|
+
/** The interval, read as `source` says. */
|
|
170
|
+
retryAfterSeconds?: number;
|
|
171
|
+
/** How long the quota is shut, as far as we can tell. */
|
|
172
|
+
waitMs: number;
|
|
173
|
+
/** How long this call has already spent waiting. */
|
|
174
|
+
waitedMs: number;
|
|
175
|
+
/** False on the last one, with `reason` saying which cap ended it. */
|
|
176
|
+
willRetry: boolean;
|
|
177
|
+
reason?: ThrottleDecision['reason'];
|
|
178
|
+
}
|
|
179
|
+
type ThrottleObserver = (event: ThrottleEvent) => void;
|
|
180
|
+
/**
|
|
181
|
+
* Watch every throttling decision in this process.
|
|
182
|
+
*
|
|
183
|
+
* One subscription rather than a logger per provider: only two of five
|
|
184
|
+
* providers had anywhere to put a log line, which is why four of them said
|
|
185
|
+
* nothing. Where the event goes is the consumer's business; that it is emitted
|
|
186
|
+
* at all is not.
|
|
187
|
+
*
|
|
188
|
+
* Pass `undefined` to stop watching. A throwing observer is ignored — a
|
|
189
|
+
* diagnostic must not turn a wait into a failure.
|
|
190
|
+
*/
|
|
191
|
+
export declare function setThrottleObserver(fn: ThrottleObserver | undefined): void;
|
|
192
|
+
/**
|
|
193
|
+
* Run `fn`, respecting the shared pause and retrying a rate limit.
|
|
194
|
+
*
|
|
195
|
+
* Any error that is not a rate limit is rethrown untouched and immediately —
|
|
196
|
+
* this is not a general-purpose retry, and turning a genuine failure into four
|
|
197
|
+
* more of them would be worse than the failure.
|
|
198
|
+
*/
|
|
199
|
+
export declare function runWithThrottleRetry<T>(fn: () => Promise<T>, opts: ThrottleRetryOptions): Promise<T>;
|
|
200
|
+
/**
|
|
201
|
+
* The rate-limit marker on an error or anywhere in its `cause` chain.
|
|
202
|
+
*
|
|
203
|
+
* Layers above the provider rewrap errors — the ILlm adapter turns anything
|
|
204
|
+
* thrown into an `LlmError` — so the marker is rarely still on the outermost
|
|
205
|
+
* object by the time a retry decorator inspects it. Bounded by depth and a
|
|
206
|
+
* visited set, like `extractStatusCode`, so a cyclic chain cannot hang.
|
|
207
|
+
*/
|
|
208
|
+
export declare function findThrottled(err: unknown): ThrottledError | undefined;
|
|
209
|
+
/**
|
|
210
|
+
* Carry the rate-limit facts onto a provider's own error.
|
|
211
|
+
*
|
|
212
|
+
* Every provider catches its transport error and rethrows something friendlier.
|
|
213
|
+
* That is fine for a message and fatal for a fact: the 429 would be lost at the
|
|
214
|
+
* exact boundary where the consumer needs it. This moves the facts across.
|
|
215
|
+
*/
|
|
216
|
+
export declare function preserveThrottled<E extends Error>(original: unknown, wrapped: E): E;
|
|
217
|
+
export {};
|
|
218
|
+
//# sourceMappingURL=throttle.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"throttle.d.ts","sourceRoot":"","sources":["../../src/llm/throttle.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,EACL,KAAK,iBAAiB,EAEtB,KAAK,gBAAgB,EACtB,MAAM,oCAAoC,CAAC;AAM5C,oFAAoF;AACpF,MAAM,WAAW,cAAe,SAAQ,KAAK;IAC3C,SAAS,EAAE,IAAI,CAAC;IAChB,gDAAgD;IAChD,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,qDAAqD;IACrD,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;;OAKG;IACH,MAAM,CAAC,EAAE,gBAAgB,CAAC,QAAQ,CAAC,CAAC;CACrC;AAED,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,IAAI,cAAc,CAMhE;AA8BD;;;;;;;;;;;GAWG;AACH,qBAAa,SAAS;IACpB,OAAO,CAAC,SAAS,CAAK;IAEtB,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,GAAE,MAAmB,GAAG,IAAI;IAIpD,wCAAwC;IACxC,SAAS,CAAC,GAAG,GAAE,MAAmB,GAAG,MAAM;IAI3C,KAAK,IAAI,IAAI;IAIb;;;;;;;;;;;OAWG;IACG,aAAa,CACjB,MAAM,CAAC,EAAE,WAAW,EACpB,SAAS,SAA2B,GACnC,OAAO,CAAC,IAAI,CAAC;CAiBjB;AAWD;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,QAAc,CAAC;AAC5C;;;;;;GAMG;AACH,eAAO,MAAM,UAAU,MAAM,CAAC;AAE9B,oFAAoF;AACpF,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAe9C;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG;IAC3C,IAAI,EAAE,SAAS,CAAC;IAChB,OAAO,EAAE,MAAM,IAAI,CAAC;CACrB,CAgBA;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,GAAG,GAAE,MAAmB,GAAG,IAAI,CAM9D;AAsCD,oEAAoE;AACpE,wBAAgB,cAAc,IAAI,MAAM,CAEvC;AAED,oCAAoC;AACpC,wBAAgB,eAAe,IAAI,IAAI,CAEtC;AAED,MAAM,WAAW,oBAAoB;IACnC,6EAA6E;IAC7E,GAAG,EAAE,MAAM,CAAC;IACZ,sDAAsD;IACtD,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAC7B,gDAAgD;IAChD,WAAW,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC;IACzC,gEAAgE;IAChE,iBAAiB,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,MAAM,GAAG,SAAS,CAAC;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,OAAO,CAAC,EAAE,CAAC,IAAI,EAAE;QACf,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;QAChB,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B,KAAK,IAAI,CAAC;CACZ;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,aAAa;IAC5B,qEAAqE;IACrE,GAAG,EAAE,MAAM,CAAC;IACZ,uCAAuC;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;;;;;;OASG;IACH,MAAM,EAAE,UAAU,GAAG,MAAM,CAAC;IAC5B,wEAAwE;IACxE,OAAO,EAAE,MAAM,CAAC;IAChB,2CAA2C;IAC3C,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,yDAAyD;IACzD,MAAM,EAAE,MAAM,CAAC;IACf,oDAAoD;IACpD,QAAQ,EAAE,MAAM,CAAC;IACjB,sEAAsE;IACtE,SAAS,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,gBAAgB,CAAC,QAAQ,CAAC,CAAC;CACrC;AAED,KAAK,gBAAgB,GAAG,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;AAIvD;;;;;;;;;;GAUG;AACH,wBAAgB,mBAAmB,CAAC,EAAE,EAAE,gBAAgB,GAAG,SAAS,GAAG,IAAI,CAE1E;AAWD;;;;;;GAMG;AACH,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACpB,IAAI,EAAE,oBAAoB,GACzB,OAAO,CAAC,CAAC,CAAC,CAOZ;AAoHD;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,OAAO,GAAG,cAAc,GAAG,SAAS,CAWtE;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,SAAS,KAAK,EAC/C,QAAQ,EAAE,OAAO,EACjB,OAAO,EAAE,CAAC,GACT,CAAC,CAaH"}
|
|
@@ -0,0 +1,441 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared handling of server-governed throttling, for every LLM provider.
|
|
3
|
+
*
|
|
4
|
+
* None of the providers handled HTTP 429 (issue #282). Each one let the status
|
|
5
|
+
* collapse into an error message, so consumers were reduced to matching the
|
|
6
|
+
* substring "429" in prose to decide whether to retry.
|
|
7
|
+
*
|
|
8
|
+
* The policy lives here rather than in a provider because the providers differ
|
|
9
|
+
* in SDK but not in the shape of the problem: a shared quota, a documented
|
|
10
|
+
* `Retry-After`, and a herd of concurrent callers that will all rediscover the
|
|
11
|
+
* same limit unless something holds them back together.
|
|
12
|
+
*
|
|
13
|
+
* The delay is always the server's. `Retry-After` may arrive as seconds or as
|
|
14
|
+
* an HTTP-date; both reduce to a delay from now, which is why "throttling" is
|
|
15
|
+
* the right word for it and why none of it is ours to invent.
|
|
16
|
+
*
|
|
17
|
+
* SAP AI Core states the rules this follows
|
|
18
|
+
* (help.sap.com, Rate Limit Management):
|
|
19
|
+
*
|
|
20
|
+
* Do not retry immediately after a 429. Use exponential backoff with jitter
|
|
21
|
+
* to prevent thundering-herd behavior. If the response includes a
|
|
22
|
+
* `Retry-After` header, use its value as the wait interval. Limit retries by
|
|
23
|
+
* using a maximum cap (for example, 5 retries or 60 seconds total wait).
|
|
24
|
+
*
|
|
25
|
+
* The same rules fit Anthropic and OpenAI, which return 429 with `Retry-After`
|
|
26
|
+
* as ordinary HTTP. Ollama is local and never rate-limited; it simply never
|
|
27
|
+
* enters this path.
|
|
28
|
+
*/
|
|
29
|
+
import { ReportThrottling, } from '../interfaces/throttle-strategy.js';
|
|
30
|
+
import { DefaultWaitStrategy } from '../interfaces/wait-strategy.js';
|
|
31
|
+
/** Nothing waits unless the consumer says so. */
|
|
32
|
+
const defaultStrategy = new ReportThrottling();
|
|
33
|
+
export function isThrottledError(e) {
|
|
34
|
+
return (typeof e === 'object' &&
|
|
35
|
+
e !== null &&
|
|
36
|
+
e.throttled === true);
|
|
37
|
+
}
|
|
38
|
+
const waiter = new DefaultWaitStrategy();
|
|
39
|
+
/**
|
|
40
|
+
* Sleep, or throw if the signal aborts.
|
|
41
|
+
*
|
|
42
|
+
* Delegated to the shared wait strategy rather than hand-rolled: the listener
|
|
43
|
+
* bookkeeping is where a sleep like this goes wrong. One added per backoff and
|
|
44
|
+
* never removed accumulates on a request- or session-scoped signal until Node
|
|
45
|
+
* warns about the leak — a defect this repository has already fixed once, in
|
|
46
|
+
* the retry decorators. Fixed in one place, it stays fixed for every waiter.
|
|
47
|
+
*/
|
|
48
|
+
const sleep = async (ms, signal) => {
|
|
49
|
+
const aborted = () => signal?.reason ?? new Error('Aborted');
|
|
50
|
+
if (signal?.aborted)
|
|
51
|
+
throw aborted();
|
|
52
|
+
if ((await waiter.wait(ms, signal)) === 'aborted')
|
|
53
|
+
throw aborted();
|
|
54
|
+
};
|
|
55
|
+
/**
|
|
56
|
+
* Per-caller spread on waking, so everyone released by one penalty does not
|
|
57
|
+
* wake in the same millisecond and rebuild the herd the penalty broke up.
|
|
58
|
+
*
|
|
59
|
+
* It is added to a wait, never to a budget: a wait capped at exactly the budget
|
|
60
|
+
* simply loses the spread, which costs nothing — a caller whose budget ends
|
|
61
|
+
* with the pause is leaving either way.
|
|
62
|
+
*/
|
|
63
|
+
const WAKE_SPREAD_MS = 250;
|
|
64
|
+
/**
|
|
65
|
+
* One shared pause per quota.
|
|
66
|
+
*
|
|
67
|
+
* A rate limit belongs to the quota, not to the request that happened to
|
|
68
|
+
* discover it. When one call is told to wait, every other call against the same
|
|
69
|
+
* quota is already over the limit too — letting each find that out for itself
|
|
70
|
+
* means N more rejected requests and N more penalties, which is how a limit
|
|
71
|
+
* that should last one window lasts several.
|
|
72
|
+
*
|
|
73
|
+
* Keyed because limits are per model: one model being throttled says nothing
|
|
74
|
+
* about another.
|
|
75
|
+
*/
|
|
76
|
+
export class QuotaGate {
|
|
77
|
+
notBefore = 0;
|
|
78
|
+
/** Hold every caller until the named time. Never shortens an existing hold. */
|
|
79
|
+
penalise(ms, now = Date.now()) {
|
|
80
|
+
this.notBefore = Math.max(this.notBefore, now + ms);
|
|
81
|
+
}
|
|
82
|
+
/** Milliseconds still to wait, or 0. */
|
|
83
|
+
remaining(now = Date.now()) {
|
|
84
|
+
return Math.max(0, this.notBefore - now);
|
|
85
|
+
}
|
|
86
|
+
clear() {
|
|
87
|
+
this.notBefore = 0;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Wait out any active hold.
|
|
91
|
+
*
|
|
92
|
+
* Re-checked after every wake, because the hold is not fixed at the moment
|
|
93
|
+
* this caller started waiting: another caller's 429 extends it, and a sleeper
|
|
94
|
+
* timed against the old deadline would wake early and spend a request on a
|
|
95
|
+
* quota the server has since said is still closed.
|
|
96
|
+
*
|
|
97
|
+
* The wait is jittered per caller: without that, everyone released by one
|
|
98
|
+
* penalty wakes in the same millisecond and rebuilds the herd the penalty was
|
|
99
|
+
* meant to break up.
|
|
100
|
+
*/
|
|
101
|
+
async waitUntilOpen(signal, maxWaitMs = Number.POSITIVE_INFINITY) {
|
|
102
|
+
const deadline = Date.now() + maxWaitMs;
|
|
103
|
+
for (;;) {
|
|
104
|
+
const left = this.remaining();
|
|
105
|
+
if (left <= 0)
|
|
106
|
+
return;
|
|
107
|
+
const budget = deadline - Date.now();
|
|
108
|
+
// Out of the caller's budget with the gate still shut. It is told, not
|
|
109
|
+
// held: `remaining()` still reports the hold, and the caller decides.
|
|
110
|
+
if (budget <= 0)
|
|
111
|
+
return;
|
|
112
|
+
// A timer that lands a hair early is simply waited out again on the next
|
|
113
|
+
// turn, which is why the cap needs no padding to be safe.
|
|
114
|
+
await sleep(Math.min(left + Math.random() * WAKE_SPREAD_MS, budget), signal);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
const gates = new Map();
|
|
119
|
+
/**
|
|
120
|
+
* Keys are open-ended — a per-request model override mints a new one — so the
|
|
121
|
+
* registry is pruned rather than left to grow for the life of the process.
|
|
122
|
+
* Only entries that are both open and long idle are dropped: a held gate is
|
|
123
|
+
* doing its job, and a recently used one may still be someone's in-flight call.
|
|
124
|
+
*/
|
|
125
|
+
export const GATE_IDLE_TTL_MS = 10 * 60_000;
|
|
126
|
+
/**
|
|
127
|
+
* How many quotas the registry keeps. Enforced by evicting gates that are both
|
|
128
|
+
* open and unheld, so a process with more than this many pauses or in-flight
|
|
129
|
+
* calls at once can exceed it — those gates are load-bearing. Evicting an idle
|
|
130
|
+
* one costs nothing: it carries no deadline, and the next call mints an
|
|
131
|
+
* equivalent.
|
|
132
|
+
*/
|
|
133
|
+
export const GATE_LIMIT = 500;
|
|
134
|
+
/** The gate for one quota. Shared across every provider instance in the process. */
|
|
135
|
+
export function gateFor(key) {
|
|
136
|
+
const now = Date.now();
|
|
137
|
+
const existing = gates.get(key);
|
|
138
|
+
if (existing) {
|
|
139
|
+
existing.lastUsed = now;
|
|
140
|
+
return existing.gate;
|
|
141
|
+
}
|
|
142
|
+
if (gates.size >= GATE_LIMIT)
|
|
143
|
+
reclaimGates(now);
|
|
144
|
+
const entry = {
|
|
145
|
+
gate: new QuotaGate(),
|
|
146
|
+
lastUsed: now,
|
|
147
|
+
leases: 0,
|
|
148
|
+
};
|
|
149
|
+
gates.set(key, entry);
|
|
150
|
+
return entry.gate;
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Hold a gate for the length of one call, and keep it in the registry meanwhile.
|
|
154
|
+
*
|
|
155
|
+
* Being open is not the same as being unused. A request is in flight for as
|
|
156
|
+
* long as it takes the server to answer, and until that answer arrives the gate
|
|
157
|
+
* it belongs to has no pause on it. Evicted in that window, the call would go on
|
|
158
|
+
* to penalise an object the registry no longer knows: the pause would be real
|
|
159
|
+
* and invisible, and the next caller would be handed a fresh open gate and sent
|
|
160
|
+
* straight into a quota that had just closed. Worse than no gate at all, since
|
|
161
|
+
* the penalty is paid and nothing is bought with it.
|
|
162
|
+
*
|
|
163
|
+
* Release in a `finally`. Releasing twice is harmless.
|
|
164
|
+
*/
|
|
165
|
+
export function leaseQuotaGate(key) {
|
|
166
|
+
const gate = gateFor(key);
|
|
167
|
+
// By identity, not by key: `resetQuotaGates` may have replaced the entry
|
|
168
|
+
// by the time this is released, and that entry's count is not ours to touch.
|
|
169
|
+
const entry = gates.get(key);
|
|
170
|
+
if (entry)
|
|
171
|
+
entry.leases += 1;
|
|
172
|
+
let released = false;
|
|
173
|
+
return {
|
|
174
|
+
gate,
|
|
175
|
+
release: () => {
|
|
176
|
+
if (released || !entry)
|
|
177
|
+
return;
|
|
178
|
+
released = true;
|
|
179
|
+
entry.leases -= 1;
|
|
180
|
+
entry.lastUsed = Date.now();
|
|
181
|
+
},
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Reclaim gates that are open and long idle. Exported so a long-lived host can
|
|
186
|
+
* run it on a timer, and so the reclaim can be tested at a chosen `now` without
|
|
187
|
+
* waiting ten minutes. It frees only what is certainly stale; the size bound is
|
|
188
|
+
* enforced separately, by `reclaimGates`.
|
|
189
|
+
*/
|
|
190
|
+
export function pruneQuotaGates(now = Date.now()) {
|
|
191
|
+
for (const [key, entry] of gates) {
|
|
192
|
+
if (isEvictable(entry, now) && now - entry.lastUsed > GATE_IDLE_TTL_MS) {
|
|
193
|
+
gates.delete(key);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* Safe to forget: nobody is waiting behind it, and no call is holding it.
|
|
199
|
+
*
|
|
200
|
+
* An open, unleased gate carries no state at all — the next call mints an
|
|
201
|
+
* equivalent one and loses nothing.
|
|
202
|
+
*/
|
|
203
|
+
function isEvictable(entry, now) {
|
|
204
|
+
return entry.leases === 0 && entry.gate.remaining(now) === 0;
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* Hold the registry at `GATE_LIMIT`.
|
|
208
|
+
*
|
|
209
|
+
* Age alone is not a bound: a burst of per-request model overrides mints keys
|
|
210
|
+
* far faster than the ten-minute idle rule retires them, and the map grows for
|
|
211
|
+
* as long as the burst lasts. So once the stale entries are gone, the least
|
|
212
|
+
* recently used OPEN gates go too, oldest first, until the registry is back
|
|
213
|
+
* under the limit.
|
|
214
|
+
*
|
|
215
|
+
* Only idle gates are evicted: not one holding a pause, which IS the pause, and
|
|
216
|
+
* not one a call is still holding, which may be about to become a pause.
|
|
217
|
+
*/
|
|
218
|
+
function reclaimGates(now) {
|
|
219
|
+
pruneQuotaGates(now);
|
|
220
|
+
if (gates.size < GATE_LIMIT)
|
|
221
|
+
return;
|
|
222
|
+
const evictable = [...gates]
|
|
223
|
+
.filter(([, entry]) => isEvictable(entry, now))
|
|
224
|
+
.sort((a, b) => a[1].lastUsed - b[1].lastUsed);
|
|
225
|
+
for (const [key] of evictable) {
|
|
226
|
+
if (gates.size < GATE_LIMIT)
|
|
227
|
+
return;
|
|
228
|
+
gates.delete(key);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
/** How many quotas are currently tracked. Diagnostics and tests. */
|
|
232
|
+
export function quotaGateCount() {
|
|
233
|
+
return gates.size;
|
|
234
|
+
}
|
|
235
|
+
/** Test seam — drops every gate. */
|
|
236
|
+
export function resetQuotaGates() {
|
|
237
|
+
gates.clear();
|
|
238
|
+
}
|
|
239
|
+
let observer;
|
|
240
|
+
/**
|
|
241
|
+
* Watch every throttling decision in this process.
|
|
242
|
+
*
|
|
243
|
+
* One subscription rather than a logger per provider: only two of five
|
|
244
|
+
* providers had anywhere to put a log line, which is why four of them said
|
|
245
|
+
* nothing. Where the event goes is the consumer's business; that it is emitted
|
|
246
|
+
* at all is not.
|
|
247
|
+
*
|
|
248
|
+
* Pass `undefined` to stop watching. A throwing observer is ignored — a
|
|
249
|
+
* diagnostic must not turn a wait into a failure.
|
|
250
|
+
*/
|
|
251
|
+
export function setThrottleObserver(fn) {
|
|
252
|
+
observer = fn;
|
|
253
|
+
}
|
|
254
|
+
function emit(event) {
|
|
255
|
+
if (!observer)
|
|
256
|
+
return;
|
|
257
|
+
try {
|
|
258
|
+
observer(event);
|
|
259
|
+
}
|
|
260
|
+
catch {
|
|
261
|
+
// An observer that throws is a broken diagnostic, not a broken request.
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* Run `fn`, respecting the shared pause and retrying a rate limit.
|
|
266
|
+
*
|
|
267
|
+
* Any error that is not a rate limit is rethrown untouched and immediately —
|
|
268
|
+
* this is not a general-purpose retry, and turning a genuine failure into four
|
|
269
|
+
* more of them would be worse than the failure.
|
|
270
|
+
*/
|
|
271
|
+
export async function runWithThrottleRetry(fn, opts) {
|
|
272
|
+
const { gate, release } = leaseQuotaGate(opts.key);
|
|
273
|
+
try {
|
|
274
|
+
return await attemptWithGate(fn, opts, gate);
|
|
275
|
+
}
|
|
276
|
+
finally {
|
|
277
|
+
release();
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
async function attemptWithGate(fn, opts, gate) {
|
|
281
|
+
const strategy = opts.strategy ?? defaultStrategy;
|
|
282
|
+
let attempt = 0;
|
|
283
|
+
let waited = 0;
|
|
284
|
+
// True when the wait ahead is one this caller already chose, in the catch
|
|
285
|
+
// below. Without it the gate would ask the strategy a second time about a
|
|
286
|
+
// refusal it had just answered, and report the same event twice.
|
|
287
|
+
let servingOwnWait = false;
|
|
288
|
+
for (;;) {
|
|
289
|
+
// A quota known to be shut is not a reason to hold this caller — it is a
|
|
290
|
+
// reason not to spend a request finding out. The strategy decides which:
|
|
291
|
+
// sit out the interval the server named, or leave now with it.
|
|
292
|
+
const shut = gate.remaining();
|
|
293
|
+
if (shut > 0) {
|
|
294
|
+
if (!servingOwnWait) {
|
|
295
|
+
const decision = strategy.decide({
|
|
296
|
+
source: 'gate',
|
|
297
|
+
attempt,
|
|
298
|
+
retryAfterSeconds: shut / 1000,
|
|
299
|
+
waitedMs: waited,
|
|
300
|
+
});
|
|
301
|
+
report(opts, strategy, {
|
|
302
|
+
source: 'gate',
|
|
303
|
+
attempt,
|
|
304
|
+
retryAfterSeconds: shut / 1000,
|
|
305
|
+
waitMs: shut,
|
|
306
|
+
waitedMs: waited,
|
|
307
|
+
decision,
|
|
308
|
+
});
|
|
309
|
+
if (!decision.retry) {
|
|
310
|
+
throw knownShut(shut, attempt, decision.reason);
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
const before = Date.now();
|
|
314
|
+
await gate.waitUntilOpen(opts.signal);
|
|
315
|
+
waited += Date.now() - before;
|
|
316
|
+
}
|
|
317
|
+
servingOwnWait = false;
|
|
318
|
+
try {
|
|
319
|
+
return await fn();
|
|
320
|
+
}
|
|
321
|
+
catch (error) {
|
|
322
|
+
if (!opts.isThrottled(error))
|
|
323
|
+
throw error;
|
|
324
|
+
attempt += 1;
|
|
325
|
+
const retryAfter = opts.retryAfterSeconds?.(error);
|
|
326
|
+
const decision = strategy.decide({
|
|
327
|
+
source: 'response',
|
|
328
|
+
attempt,
|
|
329
|
+
retryAfterSeconds: retryAfter,
|
|
330
|
+
waitedMs: waited,
|
|
331
|
+
});
|
|
332
|
+
// Recorded whether or not THIS caller goes on. What the server said is
|
|
333
|
+
// true for everyone, and a caller that leaves without writing it down
|
|
334
|
+
// sends the next one to discover it again, at the cost of another
|
|
335
|
+
// refusal. Knowing is shared; waiting is not.
|
|
336
|
+
gate.penalise(decision.waitMs);
|
|
337
|
+
report(opts, strategy, {
|
|
338
|
+
source: 'response',
|
|
339
|
+
attempt,
|
|
340
|
+
retryAfterSeconds: retryAfter,
|
|
341
|
+
waitMs: decision.waitMs,
|
|
342
|
+
waitedMs: waited,
|
|
343
|
+
decision,
|
|
344
|
+
});
|
|
345
|
+
if (!decision.retry) {
|
|
346
|
+
throw annotate(error, attempt, retryAfter, decision.reason);
|
|
347
|
+
}
|
|
348
|
+
opts.onRetry?.({
|
|
349
|
+
attempt,
|
|
350
|
+
delayMs: decision.waitMs,
|
|
351
|
+
retryAfterSeconds: retryAfter,
|
|
352
|
+
});
|
|
353
|
+
servingOwnWait = true;
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
function report(opts, strategy, info) {
|
|
358
|
+
emit({
|
|
359
|
+
key: opts.key,
|
|
360
|
+
strategy: strategy.name,
|
|
361
|
+
source: info.source,
|
|
362
|
+
attempt: info.attempt,
|
|
363
|
+
retryAfterSeconds: info.retryAfterSeconds,
|
|
364
|
+
waitMs: info.waitMs,
|
|
365
|
+
waitedMs: info.waitedMs,
|
|
366
|
+
willRetry: info.decision.retry,
|
|
367
|
+
reason: info.decision.reason,
|
|
368
|
+
});
|
|
369
|
+
}
|
|
370
|
+
const MAX_CAUSE_DEPTH = 5;
|
|
371
|
+
/**
|
|
372
|
+
* The rate-limit marker on an error or anywhere in its `cause` chain.
|
|
373
|
+
*
|
|
374
|
+
* Layers above the provider rewrap errors — the ILlm adapter turns anything
|
|
375
|
+
* thrown into an `LlmError` — so the marker is rarely still on the outermost
|
|
376
|
+
* object by the time a retry decorator inspects it. Bounded by depth and a
|
|
377
|
+
* visited set, like `extractStatusCode`, so a cyclic chain cannot hang.
|
|
378
|
+
*/
|
|
379
|
+
export function findThrottled(err) {
|
|
380
|
+
const visited = new Set();
|
|
381
|
+
let cur = err;
|
|
382
|
+
for (let depth = 0; depth < MAX_CAUSE_DEPTH; depth++) {
|
|
383
|
+
if (typeof cur !== 'object' || cur === null || visited.has(cur))
|
|
384
|
+
return undefined;
|
|
385
|
+
visited.add(cur);
|
|
386
|
+
if (isThrottledError(cur))
|
|
387
|
+
return cur;
|
|
388
|
+
cur = cur.cause;
|
|
389
|
+
}
|
|
390
|
+
return undefined;
|
|
391
|
+
}
|
|
392
|
+
/**
|
|
393
|
+
* Carry the rate-limit facts onto a provider's own error.
|
|
394
|
+
*
|
|
395
|
+
* Every provider catches its transport error and rethrows something friendlier.
|
|
396
|
+
* That is fine for a message and fatal for a fact: the 429 would be lost at the
|
|
397
|
+
* exact boundary where the consumer needs it. This moves the facts across.
|
|
398
|
+
*/
|
|
399
|
+
export function preserveThrottled(original, wrapped) {
|
|
400
|
+
if (!isThrottledError(original))
|
|
401
|
+
return wrapped;
|
|
402
|
+
const w = wrapped;
|
|
403
|
+
w.throttled = true;
|
|
404
|
+
w.attempts = original.attempts;
|
|
405
|
+
if (original.retryAfterSeconds !== undefined) {
|
|
406
|
+
w.retryAfterSeconds = original.retryAfterSeconds;
|
|
407
|
+
}
|
|
408
|
+
// Every fact, not a chosen few. Every provider wraps its transport error, so
|
|
409
|
+
// a fact left behind here is a fact no consumer ever sees — which is how the
|
|
410
|
+
// reason for giving up was lost the moment it was added.
|
|
411
|
+
if (original.reason !== undefined)
|
|
412
|
+
w.reason = original.reason;
|
|
413
|
+
return w;
|
|
414
|
+
}
|
|
415
|
+
/**
|
|
416
|
+
* The quota is shut and this caller is not waiting it out.
|
|
417
|
+
*
|
|
418
|
+
* No request was sent, because there is nothing to learn from a refusal we can
|
|
419
|
+
* already predict — and a refusal we ask for is a refusal we are charged for.
|
|
420
|
+
* The remaining interval goes up with the error so the caller can decide.
|
|
421
|
+
*/
|
|
422
|
+
function knownShut(remainingMs, attempts, reason) {
|
|
423
|
+
const e = new Error('throttled: the quota is closed and this call is not waiting it out');
|
|
424
|
+
e.throttled = true;
|
|
425
|
+
e.attempts = attempts;
|
|
426
|
+
e.retryAfterSeconds = remainingMs / 1000;
|
|
427
|
+
if (reason !== undefined)
|
|
428
|
+
e.reason = reason;
|
|
429
|
+
return e;
|
|
430
|
+
}
|
|
431
|
+
function annotate(error, attempts, retryAfterSeconds, reason) {
|
|
432
|
+
const e = (error instanceof Error ? error : new Error(String(error)));
|
|
433
|
+
e.throttled = true;
|
|
434
|
+
e.attempts = attempts;
|
|
435
|
+
if (retryAfterSeconds !== undefined)
|
|
436
|
+
e.retryAfterSeconds = retryAfterSeconds;
|
|
437
|
+
if (reason !== undefined)
|
|
438
|
+
e.reason = reason;
|
|
439
|
+
return e;
|
|
440
|
+
}
|
|
441
|
+
//# sourceMappingURL=throttle.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"throttle.js","sourceRoot":"","sources":["../../src/llm/throttle.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,EAEL,gBAAgB,GAEjB,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,gCAAgC,CAAC;AAErE,iDAAiD;AACjD,MAAM,eAAe,GAAG,IAAI,gBAAgB,EAAE,CAAC;AAkB/C,MAAM,UAAU,gBAAgB,CAAC,CAAU;IACzC,OAAO,CACL,OAAO,CAAC,KAAK,QAAQ;QACrB,CAAC,KAAK,IAAI;QACT,CAA6B,CAAC,SAAS,KAAK,IAAI,CAClD,CAAC;AACJ,CAAC;AAED,MAAM,MAAM,GAAG,IAAI,mBAAmB,EAAE,CAAC;AAEzC;;;;;;;;GAQG;AACH,MAAM,KAAK,GAAG,KAAK,EAAE,EAAU,EAAE,MAAoB,EAAiB,EAAE;IACtE,MAAM,OAAO,GAAG,GAAU,EAAE,CACzB,MAAM,EAAE,MAA4B,IAAI,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC;IAChE,IAAI,MAAM,EAAE,OAAO;QAAE,MAAM,OAAO,EAAE,CAAC;IACrC,IAAI,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,KAAK,SAAS;QAAE,MAAM,OAAO,EAAE,CAAC;AACrE,CAAC,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,cAAc,GAAG,GAAG,CAAC;AAE3B;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,SAAS;IACZ,SAAS,GAAG,CAAC,CAAC;IAEtB,+EAA+E;IAC/E,QAAQ,CAAC,EAAU,EAAE,MAAc,IAAI,CAAC,GAAG,EAAE;QAC3C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,GAAG,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,wCAAwC;IACxC,SAAS,CAAC,MAAc,IAAI,CAAC,GAAG,EAAE;QAChC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC;IAC3C,CAAC;IAED,KAAK;QACH,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;IACrB,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,aAAa,CACjB,MAAoB,EACpB,SAAS,GAAG,MAAM,CAAC,iBAAiB;QAEpC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QACxC,SAAS,CAAC;YACR,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;YAC9B,IAAI,IAAI,IAAI,CAAC;gBAAE,OAAO;YACtB,MAAM,MAAM,GAAG,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACrC,uEAAuE;YACvE,sEAAsE;YACtE,IAAI,MAAM,IAAI,CAAC;gBAAE,OAAO;YACxB,yEAAyE;YACzE,0DAA0D;YAC1D,MAAM,KAAK,CACT,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,cAAc,EAAE,MAAM,CAAC,EACvD,MAAM,CACP,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AASD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAqB,CAAC;AAE3C;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,EAAE,GAAG,MAAM,CAAC;AAC5C;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,GAAG,CAAC;AAE9B,oFAAoF;AACpF,MAAM,UAAU,OAAO,CAAC,GAAW;IACjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,QAAQ,EAAE,CAAC;QACb,QAAQ,CAAC,QAAQ,GAAG,GAAG,CAAC;QACxB,OAAO,QAAQ,CAAC,IAAI,CAAC;IACvB,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,IAAI,UAAU;QAAE,YAAY,CAAC,GAAG,CAAC,CAAC;IAChD,MAAM,KAAK,GAAc;QACvB,IAAI,EAAE,IAAI,SAAS,EAAE;QACrB,QAAQ,EAAE,GAAG;QACb,MAAM,EAAE,CAAC;KACV,CAAC;IACF,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACtB,OAAO,KAAK,CAAC,IAAI,CAAC;AACpB,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,cAAc,CAAC,GAAW;IAIxC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IAC1B,yEAAyE;IACzE,6EAA6E;IAC7E,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,KAAK;QAAE,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;IAC7B,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,OAAO;QACL,IAAI;QACJ,OAAO,EAAE,GAAG,EAAE;YACZ,IAAI,QAAQ,IAAI,CAAC,KAAK;gBAAE,OAAO;YAC/B,QAAQ,GAAG,IAAI,CAAC;YAChB,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;YAClB,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,MAAc,IAAI,CAAC,GAAG,EAAE;IACtD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,KAAK,EAAE,CAAC;QACjC,IAAI,WAAW,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,GAAG,GAAG,KAAK,CAAC,QAAQ,GAAG,gBAAgB,EAAE,CAAC;YACvE,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,SAAS,WAAW,CAAC,KAAgB,EAAE,GAAW;IAChD,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;AAC/D,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAS,YAAY,CAAC,GAAW;IAC/B,eAAe,CAAC,GAAG,CAAC,CAAC;IACrB,IAAI,KAAK,CAAC,IAAI,GAAG,UAAU;QAAE,OAAO;IAEpC,MAAM,SAAS,GAAG,CAAC,GAAG,KAAK,CAAC;SACzB,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;SAC9C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAEjD,KAAK,MAAM,CAAC,GAAG,CAAC,IAAI,SAAS,EAAE,CAAC;QAC9B,IAAI,KAAK,CAAC,IAAI,GAAG,UAAU;YAAE,OAAO;QACpC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACpB,CAAC;AACH,CAAC;AAED,oEAAoE;AACpE,MAAM,UAAU,cAAc;IAC5B,OAAO,KAAK,CAAC,IAAI,CAAC;AACpB,CAAC;AAED,oCAAoC;AACpC,MAAM,UAAU,eAAe;IAC7B,KAAK,CAAC,KAAK,EAAE,CAAC;AAChB,CAAC;AA6DD,IAAI,QAAsC,CAAC;AAE3C;;;;;;;;;;GAUG;AACH,MAAM,UAAU,mBAAmB,CAAC,EAAgC;IAClE,QAAQ,GAAG,EAAE,CAAC;AAChB,CAAC;AAED,SAAS,IAAI,CAAC,KAAoB;IAChC,IAAI,CAAC,QAAQ;QAAE,OAAO;IACtB,IAAI,CAAC;QACH,QAAQ,CAAC,KAAK,CAAC,CAAC;IAClB,CAAC;IAAC,MAAM,CAAC;QACP,wEAAwE;IAC1E,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,EAAoB,EACpB,IAA0B;IAE1B,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnD,IAAI,CAAC;QACH,OAAO,MAAM,eAAe,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IAC/C,CAAC;YAAS,CAAC;QACT,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,EAAoB,EACpB,IAA0B,EAC1B,IAAe;IAEf,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,eAAe,CAAC;IAClD,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,0EAA0E;IAC1E,0EAA0E;IAC1E,iEAAiE;IACjE,IAAI,cAAc,GAAG,KAAK,CAAC;IAE3B,SAAS,CAAC;QACR,yEAAyE;QACzE,yEAAyE;QACzE,+DAA+D;QAC/D,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAC9B,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;YACb,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC;oBAC/B,MAAM,EAAE,MAAM;oBACd,OAAO;oBACP,iBAAiB,EAAE,IAAI,GAAG,IAAI;oBAC9B,QAAQ,EAAE,MAAM;iBACjB,CAAC,CAAC;gBACH,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE;oBACrB,MAAM,EAAE,MAAM;oBACd,OAAO;oBACP,iBAAiB,EAAE,IAAI,GAAG,IAAI;oBAC9B,MAAM,EAAE,IAAI;oBACZ,QAAQ,EAAE,MAAM;oBAChB,QAAQ;iBACT,CAAC,CAAC;gBACH,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;oBACpB,MAAM,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;gBAClD,CAAC;YACH,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtC,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC;QAChC,CAAC;QACD,cAAc,GAAG,KAAK,CAAC;QAEvB,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC;gBAAE,MAAM,KAAK,CAAC;YAE1C,OAAO,IAAI,CAAC,CAAC;YACb,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC,KAAK,CAAC,CAAC;YACnD,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC;gBAC/B,MAAM,EAAE,UAAU;gBAClB,OAAO;gBACP,iBAAiB,EAAE,UAAU;gBAC7B,QAAQ,EAAE,MAAM;aACjB,CAAC,CAAC;YAEH,uEAAuE;YACvE,sEAAsE;YACtE,kEAAkE;YAClE,8CAA8C;YAC9C,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAE/B,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE;gBACrB,MAAM,EAAE,UAAU;gBAClB,OAAO;gBACP,iBAAiB,EAAE,UAAU;gBAC7B,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,QAAQ,EAAE,MAAM;gBAChB,QAAQ;aACT,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;gBACpB,MAAM,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC9D,CAAC;YAED,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO;gBACP,OAAO,EAAE,QAAQ,CAAC,MAAM;gBACxB,iBAAiB,EAAE,UAAU;aAC9B,CAAC,CAAC;YACH,cAAc,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,MAAM,CACb,IAA0B,EAC1B,QAA2B,EAC3B,IAOC;IAED,IAAI,CAAC;QACH,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,QAAQ,EAAE,QAAQ,CAAC,IAAI;QACvB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;QACzC,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK;QAC9B,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;KAC7B,CAAC,CAAC;AACL,CAAC;AAED,MAAM,eAAe,GAAG,CAAC,CAAC;AAE1B;;;;;;;GAOG;AACH,MAAM,UAAU,aAAa,CAAC,GAAY;IACxC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAW,CAAC;IACnC,IAAI,GAAG,GAAY,GAAG,CAAC;IACvB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,eAAe,EAAE,KAAK,EAAE,EAAE,CAAC;QACrD,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC;YAC7D,OAAO,SAAS,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACjB,IAAI,gBAAgB,CAAC,GAAG,CAAC;YAAE,OAAO,GAAG,CAAC;QACtC,GAAG,GAAI,GAA2B,CAAC,KAAK,CAAC;IAC3C,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAC/B,QAAiB,EACjB,OAAU;IAEV,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC;QAAE,OAAO,OAAO,CAAC;IAChD,MAAM,CAAC,GAAG,OAAsC,CAAC;IACjD,CAAC,CAAC,SAAS,GAAG,IAAI,CAAC;IACnB,CAAC,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;IAC/B,IAAI,QAAQ,CAAC,iBAAiB,KAAK,SAAS,EAAE,CAAC;QAC7C,CAAC,CAAC,iBAAiB,GAAG,QAAQ,CAAC,iBAAiB,CAAC;IACnD,CAAC;IACD,6EAA6E;IAC7E,6EAA6E;IAC7E,yDAAyD;IACzD,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS;QAAE,CAAC,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;IAC9D,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;;;;;GAMG;AACH,SAAS,SAAS,CAChB,WAAmB,EACnB,QAAgB,EAChB,MAAgC;IAEhC,MAAM,CAAC,GAAG,IAAI,KAAK,CACjB,oEAAoE,CACnD,CAAC;IACpB,CAAC,CAAC,SAAS,GAAG,IAAI,CAAC;IACnB,CAAC,CAAC,QAAQ,GAAG,QAAQ,CAAC;IACtB,CAAC,CAAC,iBAAiB,GAAG,WAAW,GAAG,IAAI,CAAC;IACzC,IAAI,MAAM,KAAK,SAAS;QAAE,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC;IAC5C,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,QAAQ,CACf,KAAc,EACd,QAAgB,EAChB,iBAA0B,EAC1B,MAAiC;IAEjC,MAAM,CAAC,GAAG,CACR,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CACxC,CAAC;IACpB,CAAC,CAAC,SAAS,GAAG,IAAI,CAAC;IACnB,CAAC,CAAC,QAAQ,GAAG,QAAQ,CAAC;IACtB,IAAI,iBAAiB,KAAK,SAAS;QAAE,CAAC,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;IAC7E,IAAI,MAAM,KAAK,SAAS;QAAE,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC;IAC5C,OAAO,CAAC,CAAC;AACX,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Core types for LLM Proxy
|
|
3
3
|
*/
|
|
4
|
+
import type { IThrottleStrategy } from './interfaces/throttle-strategy.js';
|
|
4
5
|
import type { LlmUsage } from './interfaces/types.js';
|
|
5
|
-
import type { RateLimitPolicy } from './llm/rate-limit.js';
|
|
6
6
|
export interface Message {
|
|
7
7
|
/**
|
|
8
8
|
* Message role
|
|
@@ -79,12 +79,14 @@ export interface LLMProviderConfig {
|
|
|
79
79
|
temperature?: number;
|
|
80
80
|
maxTokens?: number;
|
|
81
81
|
/**
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
85
|
-
* caller.
|
|
82
|
+
* What this provider does when a server throttles it.
|
|
83
|
+
*
|
|
84
|
+
* Omit and nothing waits: the failure comes back carrying what the server
|
|
85
|
+
* said, and the caller decides. This is the strategy itself, not a policy
|
|
86
|
+
* with our numbers in it — a bound set out here would overrule a strategy
|
|
87
|
+
* that had decided otherwise.
|
|
86
88
|
*/
|
|
87
|
-
|
|
89
|
+
whenThrottled?: IThrottleStrategy;
|
|
88
90
|
}
|
|
89
91
|
export interface LLMCallOptions {
|
|
90
92
|
/** Per-request model override. */
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mCAAmC,CAAC;AAC3E,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAEtD,MAAM,WAAW,OAAO;IACtB;;;;;;OAMG;IACH,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ,GAAG,MAAM,CAAC;IAC/C,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,iGAAiG;IACjG,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,uFAAuF;IACvF,UAAU,CAAC,EAAE,KAAK,CAAC;QACjB,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,UAAU,CAAC;QACjB,QAAQ,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,SAAS,EAAE,MAAM,CAAA;SAAE,CAAC;KAC/C,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;+EAG2E;IAC3E,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,KAAK,CAAC;QAChB,KAAK,EAAE,MAAM,CAAC;QACd,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;IACH;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,QAAQ,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,8FAA8F;IAC9F,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;;OAOG;IACH,aAAa,CAAC,EAAE,iBAAiB,CAAC;CACnC;AAED,MAAM,WAAW,cAAc;IAC7B,kCAAkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAMD,MAAM,MAAM,gBAAgB,GACxB;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAC/B;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,SAAS,EAAE,QAAQ,EAAE,CAAA;CAAE,GAC7C;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,gBAAgB,EAAE,MAAM,CAAA;CAAE,GACjE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,GAAG,YAAY,GAAG,QAAQ,GAAG,OAAO,CAAA;CAAE,CAAC"}
|
package/package.json
CHANGED