@openllmsh/sdk 0.0.0-stage → 2.10.12-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +43 -0
- package/LICENSE +202 -0
- package/README.md +142 -2
- package/assets/openllm-light.svg +18 -0
- package/assets/openllm.svg +18 -0
- package/client/chat.ts +140 -0
- package/client/devices.ts +269 -0
- package/client/http.ts +72 -0
- package/client/index.ts +129 -0
- package/client/relay-mux.ts +547 -0
- package/client/types.ts +51 -0
- package/client/vault.ts +239 -0
- package/connect/chat.ts +58 -0
- package/connect/client.ts +240 -0
- package/connect/config.ts +47 -0
- package/connect/errors.ts +121 -0
- package/connect/index.ts +23 -0
- package/connect/oauth.ts +265 -0
- package/connect/routes.ts +234 -0
- package/connect/store.ts +107 -0
- package/mcp/handle.ts +24 -0
- package/mcp/index.ts +76 -0
- package/mcp/server.ts +319 -0
- package/mcp/tools/marketplace.ts +439 -0
- package/mcp/tools/memory-client.ts +297 -0
- package/mcp/tools/memory-config.ts +32 -0
- package/mcp/tools/memory.ts +369 -0
- package/mcp/tools/model-priority.ts +88 -0
- package/mcp/tools/openllm.ts +883 -0
- package/mcp/types.ts +28 -0
- package/openapi/call.ts +90 -0
- package/openapi/index.ts +8 -0
- package/openapi/openapi.json +80213 -0
- package/openapi/operations.ts +2075 -0
- package/package.json +39 -4
- package/protocol/CONTRIBUTING.md +21 -0
- package/protocol/README.md +134 -0
- package/protocol/access.ts +186 -0
- package/protocol/agent-prompt-text.ts +32 -0
- package/protocol/agent.ts +991 -0
- package/protocol/api-key-syntax.ts +105 -0
- package/protocol/assets/openllm-light.svg +18 -0
- package/protocol/assets/openllm.svg +18 -0
- package/protocol/audio.ts +29 -0
- package/protocol/auth.ts +142 -0
- package/protocol/changelog.ts +155 -0
- package/protocol/chat.ts +458 -0
- package/protocol/client-registry.ts +356 -0
- package/protocol/config.ts +345 -0
- package/protocol/control-channel.ts +115 -0
- package/protocol/cooldown-reason.ts +78 -0
- package/protocol/cost.ts +10 -0
- package/protocol/daemon-port.ts +31 -0
- package/protocol/daemon.ts +1524 -0
- package/protocol/default-chains.ts +66 -0
- package/protocol/device-state.ts +85 -0
- package/protocol/doctor-report-local.ts +443 -0
- package/protocol/doctor-report-policy.ts +119 -0
- package/protocol/doctor-report.ts +947 -0
- package/protocol/errors.ts +14 -0
- package/protocol/execution-selection.ts +141 -0
- package/protocol/hop-grant.ts +33 -0
- package/protocol/image-edit-parse.ts +263 -0
- package/protocol/images.ts +298 -0
- package/protocol/index.ts +74 -0
- package/protocol/inference-headers.ts +3 -0
- package/protocol/keepalive.ts +69 -0
- package/protocol/local-auth.ts +300 -0
- package/protocol/local-device-control.ts +133 -0
- package/protocol/local-provider.ts +288 -0
- package/protocol/local-session.ts +118 -0
- package/protocol/marketplace.ts +187 -0
- package/protocol/media-attempt.ts +81 -0
- package/protocol/media-bytes.ts +28 -0
- package/protocol/media-default.ts +357 -0
- package/protocol/media-dimensions.ts +132 -0
- package/protocol/media-input.ts +61 -0
- package/protocol/media-model.ts +142 -0
- package/protocol/media-option-issue.ts +90 -0
- package/protocol/media-types.ts +70 -0
- package/protocol/model-match.ts +1051 -0
- package/protocol/model-settings.ts +327 -0
- package/protocol/models.ts +767 -0
- package/protocol/mux.ts +722 -0
- package/protocol/plan-surface.ts +59 -0
- package/protocol/provider-models.ts +16 -0
- package/protocol/provider-status.ts +206 -0
- package/protocol/provider-tier.ts +67 -0
- package/protocol/provider-usage.ts +263 -0
- package/protocol/providers.ts +901 -0
- package/protocol/public-usage-index.ts +354 -0
- package/protocol/realtime-generated.ts +31 -0
- package/protocol/realtime.ts +324 -0
- package/protocol/registry-pointer.ts +118 -0
- package/protocol/relay.ts +702 -0
- package/protocol/replay-session.ts +11 -0
- package/protocol/responses.ts +280 -0
- package/protocol/semver.ts +182 -0
- package/protocol/setup-mode.ts +22 -0
- package/protocol/sha256-hex.ts +98 -0
- package/protocol/stats.ts +406 -0
- package/protocol/streaming.ts +19 -0
- package/protocol/subscription-daemon-floors.ts +33 -0
- package/protocol/subscription-provider-compat.ts +104 -0
- package/protocol/subscription-provider.ts +32 -0
- package/protocol/team-grant.ts +17 -0
- package/protocol/team-shareable.ts +61 -0
- package/protocol/transcription-encoding.ts +76 -0
- package/protocol/url-locality-dns.ts +37 -0
- package/protocol/url-locality.ts +288 -0
- package/protocol/usage-snapshots.ts +9 -0
- package/protocol/vendor-cli-install.ts +29 -0
- package/protocol/video-job-plan.ts +50 -0
- package/protocol/videos.ts +227 -0
- package/tsconfig.json +14 -0
- package/tunnel/ARCHITECTURE.md +111 -0
- package/tunnel/LICENSE +33 -0
- package/tunnel/README.md +45 -0
- package/tunnel/assets/openllm-light.svg +18 -0
- package/tunnel/assets/openllm.svg +18 -0
- package/tunnel/channel-envelope.ts +139 -0
- package/tunnel/codec.ts +191 -0
- package/tunnel/device-anchor.ts +229 -0
- package/tunnel/device-control.ts +118 -0
- package/tunnel/device-grant.ts +173 -0
- package/tunnel/fleet.ts +77 -0
- package/tunnel/index.ts +22 -0
- package/tunnel/mux.ts +580 -0
- package/tunnel/peer-link.ts +390 -0
- package/tunnel/rtc-auth.ts +415 -0
- package/tunnel/rtc-duplex.ts +187 -0
- package/tunnel/rtc-ice.ts +136 -0
- package/tunnel/secure.ts +1298 -0
- package/tunnel/stream-reset-error.ts +14 -0
- package/tunnel/streams.ts +1430 -0
- package/tunnel/team-grant.ts +302 -0
- package/tunnel/wire-bytes.ts +102 -0
- package/vault/ARCHITECTURE.md +470 -0
- package/vault/LICENSE +33 -0
- package/vault/api-key.ts +190 -0
- package/vault/argon2.ts +106 -0
- package/vault/cli-auth-seal.ts +157 -0
- package/vault/cred-seal.ts +146 -0
- package/vault/credential.ts +55 -0
- package/vault/crypto.ts +210 -0
- package/vault/daemon-attest.ts +229 -0
- package/vault/derive.ts +113 -0
- package/vault/device-access.ts +87 -0
- package/vault/device-anchor.ts +185 -0
- package/vault/device-control.ts +174 -0
- package/vault/handoff.ts +396 -0
- package/vault/index.ts +21 -0
- package/vault/passkey.ts +243 -0
- package/vault/phrase.ts +65 -0
- package/vault/pin.ts +153 -0
- package/vault/public-key-seal.ts +131 -0
- package/vault/seed.ts +272 -0
- package/vault/team.ts +355 -0
- package/vault/vault-item.ts +131 -0
- package/vault/vault-migrate-digest.ts +29 -0
- package/vault/wrapped.ts +77 -0
- package/wire/ARCHITECTURE.md +90 -0
- package/wire/CONTRIBUTING.md +19 -0
- package/wire/LICENSE +33 -0
- package/wire/README.md +51 -0
- package/wire/adapters/messages/anthropic-map.ts +37 -0
- package/wire/adapters/messages/index.ts +7 -0
- package/wire/adapters/messages/reasoning-from-items.ts +38 -0
- package/wire/adapters/messages/reasoning-signature.ts +243 -0
- package/wire/adapters/messages/request.ts +574 -0
- package/wire/adapters/messages/response.ts +215 -0
- package/wire/adapters/messages/streaming.ts +1027 -0
- package/wire/adapters/messages/usage.ts +43 -0
- package/wire/adapters/responses/index.ts +3 -0
- package/wire/adapters/responses/request.ts +395 -0
- package/wire/adapters/responses/response.ts +88 -0
- package/wire/adapters/responses/streaming.ts +344 -0
- package/wire/assets/openllm-light.svg +18 -0
- package/wire/assets/openllm.svg +18 -0
- package/wire/features/account-rank.ts +166 -0
- package/wire/features/compaction/compact-request.ts +862 -0
- package/wire/features/compaction/compaction-text.ts +43 -0
- package/wire/features/context-demote.ts +47 -0
- package/wire/features/context-overflow-strategy.ts +26 -0
- package/wire/features/context-skip.ts +170 -0
- package/wire/features/conversation-affinity.ts +144 -0
- package/wire/features/max-tokens-backfill.ts +141 -0
- package/wire/features/model-caps.ts +53 -0
- package/wire/features/model-settings.ts +24 -0
- package/wire/features/output-limit-clamp.ts +64 -0
- package/wire/features/quota-gate.ts +85 -0
- package/wire/index.ts +44 -0
- package/wire/lib/attempt-outcome.ts +43 -0
- package/wire/lib/canonical/content-part.ts +38 -0
- package/wire/lib/canonical/count-tokens-capability.ts +138 -0
- package/wire/lib/canonical/encoding-select.ts +82 -0
- package/wire/lib/canonical/image-signature.ts +202 -0
- package/wire/lib/canonical/image-sse.ts +216 -0
- package/wire/lib/canonical/message.ts +43 -0
- package/wire/lib/canonical/token-estimate.ts +266 -0
- package/wire/lib/encrypted-content.ts +66 -0
- package/wire/lib/error-class.ts +154 -0
- package/wire/lib/error-envelope.ts +56 -0
- package/wire/lib/forwarded-headers.ts +180 -0
- package/wire/lib/provider-error-rules.ts +48 -0
- package/wire/lib/refusal.ts +38 -0
- package/wire/lib/responses-apply-patch.ts +104 -0
- package/wire/lib/responses-namespace.ts +53 -0
- package/wire/lib/secret-scrub.ts +101 -0
- package/wire/lib/streaming/accumulate.ts +221 -0
- package/wire/lib/streaming/finish-reason.ts +33 -0
- package/wire/lib/streaming/peek.ts +154 -0
- package/wire/lib/streaming/provider-decode.ts +259 -0
- package/wire/lib/streaming/response-stream.ts +71 -0
- package/wire/lib/streaming/sse.ts +699 -0
- package/wire/lib/streaming/strip-tool-calls.ts +95 -0
- package/wire/lib/streaming/upstream-error.ts +58 -0
- package/wire/lib/strip-subagent-isolation.ts +284 -0
- package/wire/lib/thinking-signature.ts +156 -0
- package/wire/lib/token-ruler/bpe-core.ts +426 -0
- package/wire/lib/token-ruler/index.ts +2 -0
- package/wire/lib/token-ruler/ruler.ts +79 -0
- package/wire/lib/token-ruler/vocab/claude-ranks.ts +16 -0
- package/wire/lib/token-ruler/vocab/claude.ts +22 -0
- package/wire/lib/token-ruler/vocab/o200k-ranks.ts +18 -0
- package/wire/lib/token-ruler/vocab/o200k.ts +19 -0
- package/wire/lib/tool-name-map.ts +271 -0
- package/wire/lib/tool-schema.ts +353 -0
- package/wire/lib/usage-observer.ts +125 -0
- package/wire/providers/anthropic/adaptive-thinking.ts +291 -0
- package/wire/providers/anthropic/beta-headers.ts +165 -0
- package/wire/providers/anthropic/index.ts +10 -0
- package/wire/providers/anthropic/request.ts +607 -0
- package/wire/providers/anthropic/response.ts +155 -0
- package/wire/providers/anthropic/streaming.ts +315 -0
- package/wire/providers/anthropic/web-search-domains.ts +49 -0
- package/wire/providers/chatgpt/common.ts +136 -0
- package/wire/providers/chatgpt/index.ts +9 -0
- package/wire/providers/chatgpt/request.ts +959 -0
- package/wire/providers/chatgpt/streaming.ts +1007 -0
- package/wire/providers/chatgpt/web-search.ts +63 -0
- package/wire/providers/grok/web-search.ts +87 -0
- package/wire/providers/kimi/web-search.ts +145 -0
- package/wire/providers/upstream-deny.ts +355 -0
- package/wire/providers/upstream-request.ts +619 -0
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { TAnthropicResponse } from "@openllmsh/sdk/protocol";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Claude Code compaction (`Xm()` in bundled CLI) rejects summaries when the
|
|
5
|
+
* extracted visible text is empty or **shorter than 10 characters** after
|
|
6
|
+
* trim — see https://github.com/anthropics/claude-code/issues/11095
|
|
7
|
+
*/
|
|
8
|
+
export const COMPACTION_MIN_VISIBLE_TEXT_CHARS = 10;
|
|
9
|
+
|
|
10
|
+
export type TAnthropicAssistantVisibleTextOpts = {
|
|
11
|
+
/** Join between consecutive `text` blocks. Default `""` (matches Xm / #11095). */
|
|
12
|
+
readonly betweenBlocks?: string;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
export const anthropicAssistantVisibleTextTrimmed = (
|
|
16
|
+
resp: TAnthropicResponse,
|
|
17
|
+
opts: TAnthropicAssistantVisibleTextOpts = {},
|
|
18
|
+
): string => {
|
|
19
|
+
const betweenBlocks = opts.betweenBlocks ?? "";
|
|
20
|
+
const trimParts = betweenBlocks !== "";
|
|
21
|
+
const texts = resp.content
|
|
22
|
+
.filter((b): b is { type: "text"; text: string } => b.type === "text")
|
|
23
|
+
.map((b) => (trimParts ? b.text.trim() : b.text));
|
|
24
|
+
return texts.join(betweenBlocks).trim();
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
const FALLBACK_NO_SUMMARY =
|
|
28
|
+
"OpenLLM: no visible summary text was returned for this compaction; you may retry /compact or continue the session.";
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Ensures assistant `text` blocks meet Claude Code’s minimum length so
|
|
32
|
+
* `/compact` does not fail at ~30–40% with “did not contain valid text content”.
|
|
33
|
+
*/
|
|
34
|
+
export const ensureCompactionSafeVisibleText = (text: string): string => {
|
|
35
|
+
const t = text.trim();
|
|
36
|
+
if (t.length >= COMPACTION_MIN_VISIBLE_TEXT_CHARS) {
|
|
37
|
+
return t;
|
|
38
|
+
}
|
|
39
|
+
if (t.length === 0) {
|
|
40
|
+
return FALLBACK_NO_SUMMARY;
|
|
41
|
+
}
|
|
42
|
+
return `${t}${".".repeat(COMPACTION_MIN_VISIBLE_TEXT_CHARS - t.length)}`;
|
|
43
|
+
};
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Choose the next chain hop that can accommodate an input of the required
|
|
3
|
+
* size. Shared by the cloud dispatch chain and the coreless daemon walker so
|
|
4
|
+
* context-overflow demotion cannot drift between the two paths.
|
|
5
|
+
*/
|
|
6
|
+
export const nextLargerContextModel = (
|
|
7
|
+
chainModels: ReadonlyArray<string>,
|
|
8
|
+
currentModel: string,
|
|
9
|
+
requiredTokens: number,
|
|
10
|
+
windowOf: (modelId: string) => number | null,
|
|
11
|
+
): string | null => {
|
|
12
|
+
const currentIndex = chainModels.indexOf(currentModel);
|
|
13
|
+
if (currentIndex === -1) return null;
|
|
14
|
+
for (const modelId of chainModels.slice(currentIndex + 1)) {
|
|
15
|
+
const window = windowOf(modelId);
|
|
16
|
+
if (window !== null && window > requiredTokens) return modelId;
|
|
17
|
+
}
|
|
18
|
+
return null;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Extract the upstream tokenizer's authoritative request size from an overflow
|
|
23
|
+
* diagnostic. Recognises the three vendor shapes observed live:
|
|
24
|
+
* - vLLM-style: "maximum prompt length is N ... contains M tokens" → M
|
|
25
|
+
* - Anthropic: "prompt is too long: N tokens > M maximum" → N
|
|
26
|
+
* - Kimi/DeepSeek:"... token limit: X (requested: N)" → N
|
|
27
|
+
* Returns the REQUIRED size (how big the request actually is), not the window.
|
|
28
|
+
*/
|
|
29
|
+
export const contextOverflowRequiredTokens = (raw: string): number | null => {
|
|
30
|
+
const patterns = [
|
|
31
|
+
// vLLM-style: the "contains M tokens" count.
|
|
32
|
+
/maximum (?:prompt|context) length is\s+[\d,]+.*?contains\s+([\d,]+)\s+tokens/i,
|
|
33
|
+
// Anthropic: "prompt is too long: N tokens > M maximum".
|
|
34
|
+
/prompt is too long:\s*([\d,]+)\s*tokens/i,
|
|
35
|
+
// Kimi/DeepSeek: "token limit: X (requested: N)". The "token limit" prefix
|
|
36
|
+
// is required so an unrelated "(requested: N)" (e.g. a quota / validation
|
|
37
|
+
// error) is NOT misread as a context overflow.
|
|
38
|
+
/token limit:?\s*[\d,]+\s*\(requested:\s*([\d,]+)\)/i,
|
|
39
|
+
];
|
|
40
|
+
for (const pattern of patterns) {
|
|
41
|
+
const match = pattern.exec(raw);
|
|
42
|
+
if (match === null) continue;
|
|
43
|
+
const required = Number((match[1] ?? "").replaceAll(",", ""));
|
|
44
|
+
if (Number.isSafeInteger(required) && required > 0) return required;
|
|
45
|
+
}
|
|
46
|
+
return null;
|
|
47
|
+
};
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { TContextOverflowStrategy as TProtocolContextOverflowStrategy } from "@openllmsh/sdk/protocol";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Shared context-overflow routing strategy. The cloud and daemon resolve an
|
|
5
|
+
* absent or unrecognised persisted value to the historical hop behaviour.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
export type TContextOverflowStrategy = TProtocolContextOverflowStrategy;
|
|
9
|
+
|
|
10
|
+
export const DEFAULT_CONTEXT_OVERFLOW_STRATEGY: TContextOverflowStrategy =
|
|
11
|
+
"hop_to_larger_context";
|
|
12
|
+
|
|
13
|
+
export const resolveContextOverflowStrategy = (
|
|
14
|
+
raw: string | undefined | null,
|
|
15
|
+
): TContextOverflowStrategy =>
|
|
16
|
+
raw === "compact_in_place"
|
|
17
|
+
? "compact_in_place"
|
|
18
|
+
: DEFAULT_CONTEXT_OVERFLOW_STRATEGY;
|
|
19
|
+
|
|
20
|
+
/** Hop mode both demotes on overflow and pre-skips known-too-small hops. */
|
|
21
|
+
export const isHopToLargerContext = (
|
|
22
|
+
strategy: TContextOverflowStrategy,
|
|
23
|
+
): boolean => strategy === "hop_to_larger_context";
|
|
24
|
+
|
|
25
|
+
export const shouldDemoteOnContextOverflow = isHopToLargerContext;
|
|
26
|
+
export const shouldSkipHopForSize = isHopToLargerContext;
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Context-skip gate — the per-hop context decision, shared verbatim by
|
|
3
|
+
* the cloud dispatch chain (BYOK) and the daemon walker (device) so the
|
|
4
|
+
* two paths cannot drift.
|
|
5
|
+
*
|
|
6
|
+
* The context "ladder" is deliberately thin (reference-proxy parity —
|
|
7
|
+
* ref/CLIProxyAPI does no gateway-side rewriting either):
|
|
8
|
+
*
|
|
9
|
+
* Plan A — serve correct per-model context metadata (and pass through
|
|
10
|
+
* Codex's own `/responses/compact`) so the CLIENT compacts
|
|
11
|
+
* itself with a STABLE prefix — the only compaction that
|
|
12
|
+
* preserves prompt-cache affinity;
|
|
13
|
+
* gate — THIS module: skip a hop only when the estimate CLEARLY
|
|
14
|
+
* exceeds its window (saves the doomed round trip + prefill
|
|
15
|
+
* wait);
|
|
16
|
+
* walk — the pre-commit first-event peek
|
|
17
|
+
* (`lib/streaming/peek.ts`): the real upstream tokenizer gets
|
|
18
|
+
* the final word, and a pre-output rejection walks the chain
|
|
19
|
+
* with zero double-spend.
|
|
20
|
+
*
|
|
21
|
+
* Gateway-side tool-output rewriting (the former "Plan B") was removed
|
|
22
|
+
* on purpose: estimator-driven per-request rewrites destabilise the
|
|
23
|
+
* conversation prefix (prompt-cache misses → quota drain) and silently
|
|
24
|
+
* degrade requests the vendor would have accepted.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Confidence multiplier before a hop is abandoned over context. The
|
|
29
|
+
* routing estimator is intentionally conservative (it counts every string
|
|
30
|
+
* value), so an estimate above the catalogued input budget is enough to skip
|
|
31
|
+
* a non-final hop: that model has already advertised it cannot accept the
|
|
32
|
+
* request. Final and unknown-limit hops still reach the real tokenizer.
|
|
33
|
+
*
|
|
34
|
+
* A factor above 1 let known-over-budget requests through to an inevitable
|
|
35
|
+
* upstream context rejection. Keep this at one unless the estimator contract
|
|
36
|
+
* itself changes.
|
|
37
|
+
*/
|
|
38
|
+
export const CONTEXT_SKIP_CONFIDENCE_FACTOR = 1;
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Conservative ruler→vendor-token calibration for last-resort compaction. The
|
|
42
|
+
* local `estimateBodyTokens` ruler can count fewer tokens than a vendor's own
|
|
43
|
+
* tokenizer, so target every provider below its advertised window until the
|
|
44
|
+
* vendor-grounded overflow path corrects from a real rejection.
|
|
45
|
+
*/
|
|
46
|
+
// One shared constant across providers (collapsed from the old per-provider
|
|
47
|
+
// table). 1.2 is a moderate calibration: it does not over-compact grok/openai
|
|
48
|
+
// (higher values shrink the target and dull long contexts), and the
|
|
49
|
+
// vendor-grounded overflow path corrects the rare Claude first-pass rejection.
|
|
50
|
+
export const TOKEN_ESTIMATE_FACTOR = 1.2;
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* The compaction target for a hop: its input window shrunk by the shared
|
|
54
|
+
* calibration factor (so a body compacted to this size fits the vendor's real
|
|
55
|
+
* tokenizer, not just our ruler) and by the shared context-gate safety factor.
|
|
56
|
+
* Single source for BOTH seams (cloud dispatch chain + daemon walker) so the
|
|
57
|
+
* target can't drift.
|
|
58
|
+
*/
|
|
59
|
+
export const compactionTargetFor = (
|
|
60
|
+
_provider: string,
|
|
61
|
+
window: number,
|
|
62
|
+
): number =>
|
|
63
|
+
Math.floor((window / TOKEN_ESTIMATE_FACTOR) * CONTEXT_SKIP_CONFIDENCE_FACTOR);
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Single safety margin applied to the vendor-grounded compaction target: aim
|
|
67
|
+
* the shrunk body at `window × SAFETY` implied vendor tokens, leaving a small
|
|
68
|
+
* headroom band so the retry clears the real tokenizer even if the ratio drifts
|
|
69
|
+
* slightly between passes. ONE knob for every provider — the calibration itself
|
|
70
|
+
* is self-derived from the vendor's own numbers, so there is no per-model
|
|
71
|
+
* branching here.
|
|
72
|
+
*/
|
|
73
|
+
export const COMPACTION_OVERFLOW_SAFETY = 0.92;
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* A sane lower bound for any computed compaction target. A pathological vendor
|
|
77
|
+
* ratio (or a tiny window) must never drive the target to zero / negative — the
|
|
78
|
+
* cut would then strip the live turn too. 512 tokens comfortably holds the
|
|
79
|
+
* system prompt + the final user turn the compactor always preserves.
|
|
80
|
+
*/
|
|
81
|
+
export const COMPACTION_TARGET_FLOOR = 512;
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Vendor-grounded, self-calibrating compaction target. When the upstream
|
|
85
|
+
* rejected a body on size it reports two ground-truth numbers: `requiredTokens`
|
|
86
|
+
* (how many tokens ITS tokenizer counted for the body we sent) and, implicitly,
|
|
87
|
+
* the `window` it must fit. Our local ruler measured that same body at
|
|
88
|
+
* `localEstimate`, so the observed ratio `r = requiredTokens / localEstimate`
|
|
89
|
+
* is exact calibration for THIS conversation's content — no static per-provider
|
|
90
|
+
* factor needed.
|
|
91
|
+
*
|
|
92
|
+
* We return a target in LOCAL ruler space: shrinking the body to
|
|
93
|
+
* `floor(window / r × SAFETY)` local tokens implies a vendor count of
|
|
94
|
+
* `<= window × SAFETY`, i.e. inside the window with headroom. The result is
|
|
95
|
+
* clamped to be no looser than `compactionTargetFor(provider, window)` (the
|
|
96
|
+
* static target is an upper bound — vendor grounding may only tighten) and never
|
|
97
|
+
* below `COMPACTION_TARGET_FLOOR`. When the vendor gave no usable count we fall
|
|
98
|
+
* back to the static target. Generic across grok / chatgpt / anthropic / kimi —
|
|
99
|
+
* whatever numbers the vendor surfaces drive the cut.
|
|
100
|
+
*/
|
|
101
|
+
export const compactionTargetFromOverflow = (params: {
|
|
102
|
+
readonly requiredTokens: number | null;
|
|
103
|
+
readonly window: number;
|
|
104
|
+
readonly localEstimate: number;
|
|
105
|
+
readonly provider: string;
|
|
106
|
+
}): number => {
|
|
107
|
+
const staticTarget = compactionTargetFor(params.provider, params.window);
|
|
108
|
+
if (
|
|
109
|
+
params.requiredTokens === null ||
|
|
110
|
+
params.requiredTokens <= 0 ||
|
|
111
|
+
params.localEstimate <= 0
|
|
112
|
+
) {
|
|
113
|
+
return staticTarget;
|
|
114
|
+
}
|
|
115
|
+
const ratio = params.requiredTokens / params.localEstimate;
|
|
116
|
+
const vendorGrounded = Math.floor(
|
|
117
|
+
(params.window / ratio) * COMPACTION_OVERFLOW_SAFETY,
|
|
118
|
+
);
|
|
119
|
+
// Vendor-grounded may only TIGHTEN the static target. The floor is normally
|
|
120
|
+
// COMPACTION_TARGET_FLOOR, but a small live `context_window` can make the
|
|
121
|
+
// static target itself dip below that floor — clamping up to 512 would then
|
|
122
|
+
// return a target LOOSER than the window allows. Cap the floor at the static
|
|
123
|
+
// target so the result is always `<= staticTarget`.
|
|
124
|
+
const floor = Math.min(COMPACTION_TARGET_FLOOR, staticTarget);
|
|
125
|
+
return Math.max(floor, Math.min(staticTarget, vendorGrounded));
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Hard cap on last-resort compaction rounds (daemon walker + cloud dispatch
|
|
130
|
+
* chain). The vendor-grounded ratio converges fast (usually one tighter round
|
|
131
|
+
* after a static undercorrect), so three rounds is ample headroom; the cap's
|
|
132
|
+
* real job is to make an infinite compaction loop structurally impossible when
|
|
133
|
+
* an upstream keeps rejecting. Single source of truth — both seams import this.
|
|
134
|
+
*/
|
|
135
|
+
export const MAX_LAST_RESORT_COMPACTION_ROUNDS = 3;
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* A confirmed overflow always forces a real shrink. If the calibrated target
|
|
139
|
+
* is already below the current local estimate, use it; otherwise tighten below
|
|
140
|
+
* the current body (`localEstimate × SAFETY`, floored) so the cut is never a
|
|
141
|
+
* no-op re-send of the same bytes. Shared by the daemon walker and the cloud
|
|
142
|
+
* dispatch chain so the force-shrink math cannot drift between them.
|
|
143
|
+
*/
|
|
144
|
+
export const forcedCompactionTarget = (
|
|
145
|
+
calibrated: number,
|
|
146
|
+
localEstimate: number,
|
|
147
|
+
): number =>
|
|
148
|
+
calibrated < localEstimate
|
|
149
|
+
? calibrated
|
|
150
|
+
: Math.max(
|
|
151
|
+
COMPACTION_TARGET_FLOOR,
|
|
152
|
+
Math.floor(localEstimate * COMPACTION_OVERFLOW_SAFETY),
|
|
153
|
+
);
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Should this hop be skipped for context? True only when a later hop
|
|
157
|
+
* remains (the FINAL hop always serves — the real tokenizer must get the
|
|
158
|
+
* last word, never the heuristic estimator), the model's input budget is
|
|
159
|
+
* known, and the conservative routing estimate exceeds it. An unknown
|
|
160
|
+
* limit always serves.
|
|
161
|
+
*/
|
|
162
|
+
export const shouldSkipHopForContext = (params: {
|
|
163
|
+
readonly estimatedTokens: number;
|
|
164
|
+
readonly inputTokenLimit: number | null;
|
|
165
|
+
readonly finalHop: boolean;
|
|
166
|
+
}): boolean =>
|
|
167
|
+
!params.finalHop &&
|
|
168
|
+
params.inputTokenLimit !== null &&
|
|
169
|
+
params.estimatedTokens >
|
|
170
|
+
params.inputTokenLimit * CONTEXT_SKIP_CONFIDENCE_FACTOR;
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Conversation affinity — keep one conversation on the backend that last
|
|
3
|
+
* served it, because switching account, supplier or device cold-misses the
|
|
4
|
+
* provider's prompt cache.
|
|
5
|
+
*
|
|
6
|
+
* Every routing layer ranks its candidates (chain hops, personal accounts,
|
|
7
|
+
* team suppliers, supplier devices) from live data that drifts between turns.
|
|
8
|
+
* Affinity sits on top of that ranking: the remembered candidate moves to the
|
|
9
|
+
* front whenever it is still eligible, and the ranking only decides for a new
|
|
10
|
+
* conversation or after the remembered one is excluded (cooling, exhausted,
|
|
11
|
+
* offline, critical failure). Whatever serves next is remembered in its place.
|
|
12
|
+
*
|
|
13
|
+
* Pure: the key derivation, the reorder and an in-process TTL memory. Hosts
|
|
14
|
+
* that need a cross-instance store wrap their own behind the same get/set.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
/** Longer than any provider's prompt-cache lifetime. */
|
|
18
|
+
export const CONVERSATION_AFFINITY_TTL_MS = 60 * 60 * 1000;
|
|
19
|
+
const DEFAULT_MAX_ENTRIES = 10_000;
|
|
20
|
+
|
|
21
|
+
type TJson = Record<string, unknown>;
|
|
22
|
+
|
|
23
|
+
const isRecord = (value: unknown): value is TJson =>
|
|
24
|
+
typeof value === "object" && value !== null && !Array.isArray(value);
|
|
25
|
+
|
|
26
|
+
const nonEmpty = (value: unknown): string | null =>
|
|
27
|
+
typeof value === "string" && value.trim() !== "" ? value : null;
|
|
28
|
+
|
|
29
|
+
/** 64-bit FNV-1a (two 32-bit lanes) — a routing key, not a secret. */
|
|
30
|
+
const hash64 = (text: string): string => {
|
|
31
|
+
let a = 0x811c9dc5;
|
|
32
|
+
let b = 0x01000193 ^ 0x9e3779b9;
|
|
33
|
+
for (let i = 0; i < text.length; i += 1) {
|
|
34
|
+
const code = text.charCodeAt(i);
|
|
35
|
+
a = Math.imul(a ^ code, 0x01000193);
|
|
36
|
+
b = Math.imul(b ^ code, 0x01000193 ^ 0x5bd1e995);
|
|
37
|
+
}
|
|
38
|
+
return (
|
|
39
|
+
(a >>> 0).toString(16).padStart(8, "0") +
|
|
40
|
+
(b >>> 0).toString(16).padStart(8, "0")
|
|
41
|
+
);
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
/** The turns of an inbound body, whichever surface it arrived on. */
|
|
45
|
+
const turnsOf = (body: TJson): ReadonlyArray<unknown> => {
|
|
46
|
+
if (Array.isArray(body.messages)) return body.messages;
|
|
47
|
+
if (Array.isArray(body.input)) return body.input;
|
|
48
|
+
return [];
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* A stable key for the conversation `body` belongs to, or null when the body
|
|
53
|
+
* carries nothing to key on. A client-supplied cache/session id wins (OpenAI
|
|
54
|
+
* `prompt_cache_key`, Anthropic `metadata.user_id`, which Claude Code scopes
|
|
55
|
+
* per session); otherwise the hash of the conversation's fixed prefix — the
|
|
56
|
+
* system prompt and the turns up to and including the first user turn, which
|
|
57
|
+
* never change as the conversation grows.
|
|
58
|
+
*/
|
|
59
|
+
export const conversationAffinityKey = (body: unknown): string | null => {
|
|
60
|
+
if (!isRecord(body)) return null;
|
|
61
|
+
const cacheKey = nonEmpty(body.prompt_cache_key);
|
|
62
|
+
if (cacheKey !== null) return `k:${hash64(cacheKey)}`;
|
|
63
|
+
const metadata = isRecord(body.metadata) ? body.metadata : null;
|
|
64
|
+
const sessionId = nonEmpty(metadata?.user_id);
|
|
65
|
+
if (sessionId !== null) return `s:${hash64(sessionId)}`;
|
|
66
|
+
const turns = turnsOf(body);
|
|
67
|
+
const firstUser = turns.findIndex(
|
|
68
|
+
(turn) => isRecord(turn) && turn.role === "user",
|
|
69
|
+
);
|
|
70
|
+
// A bare string `input` (Responses) is a one-shot with no history to keep.
|
|
71
|
+
if (firstUser === -1) return null;
|
|
72
|
+
const prefix = JSON.stringify([
|
|
73
|
+
body.system ?? null,
|
|
74
|
+
body.instructions ?? null,
|
|
75
|
+
turns.slice(0, firstUser + 1),
|
|
76
|
+
]);
|
|
77
|
+
return `p:${hash64(prefix)}`;
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
/** `items` with the remembered one first; unchanged when it isn't present. */
|
|
81
|
+
export const preferRemembered = <T>(
|
|
82
|
+
items: ReadonlyArray<T>,
|
|
83
|
+
isRemembered: (item: T) => boolean,
|
|
84
|
+
): ReadonlyArray<T> => {
|
|
85
|
+
const index = items.findIndex(isRemembered);
|
|
86
|
+
if (index <= 0) return items;
|
|
87
|
+
return [
|
|
88
|
+
items[index] as T,
|
|
89
|
+
...items.slice(0, index),
|
|
90
|
+
...items.slice(index + 1),
|
|
91
|
+
];
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
export type TAffinityMemory = {
|
|
95
|
+
readonly get: (key: string) => string | undefined;
|
|
96
|
+
readonly set: (key: string, value: string) => void;
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
/** Bounded in-process memory; the oldest entry is evicted first. */
|
|
100
|
+
export const createAffinityMemory = (options?: {
|
|
101
|
+
readonly ttlMs?: number;
|
|
102
|
+
readonly maxEntries?: number;
|
|
103
|
+
readonly now?: () => number;
|
|
104
|
+
}): TAffinityMemory => {
|
|
105
|
+
const ttlMs = options?.ttlMs ?? CONVERSATION_AFFINITY_TTL_MS;
|
|
106
|
+
const maxEntries = options?.maxEntries ?? DEFAULT_MAX_ENTRIES;
|
|
107
|
+
const now = options?.now ?? Date.now;
|
|
108
|
+
const entries = new Map<string, { value: string; expiresAt: number }>();
|
|
109
|
+
return {
|
|
110
|
+
get: (key) => {
|
|
111
|
+
const entry = entries.get(key);
|
|
112
|
+
if (entry === undefined) return undefined;
|
|
113
|
+
if (entry.expiresAt <= now()) {
|
|
114
|
+
entries.delete(key);
|
|
115
|
+
return undefined;
|
|
116
|
+
}
|
|
117
|
+
return entry.value;
|
|
118
|
+
},
|
|
119
|
+
set: (key, value) => {
|
|
120
|
+
entries.delete(key);
|
|
121
|
+
entries.set(key, { value, expiresAt: now() + ttlMs });
|
|
122
|
+
while (entries.size > maxEntries) {
|
|
123
|
+
const oldest = entries.keys().next().value;
|
|
124
|
+
if (oldest === undefined) break;
|
|
125
|
+
entries.delete(oldest);
|
|
126
|
+
}
|
|
127
|
+
},
|
|
128
|
+
};
|
|
129
|
+
};
|
|
130
|
+
|
|
131
|
+
/** Memory slots, one per routing decision a conversation pins. */
|
|
132
|
+
export const affinitySlot = {
|
|
133
|
+
/** The chain hop (model id) that served. */
|
|
134
|
+
hop: (conversation: string): string => `${conversation}|hop`,
|
|
135
|
+
/** The personal account index that served `provider`. */
|
|
136
|
+
account: (conversation: string, provider: string): string =>
|
|
137
|
+
`${conversation}|account|${provider}`,
|
|
138
|
+
/** The team share that served `teamModelId`. */
|
|
139
|
+
share: (conversation: string, teamModelId: string): string =>
|
|
140
|
+
`${conversation}|share|${teamModelId}`,
|
|
141
|
+
/** The supplier device that served `shareId`. */
|
|
142
|
+
device: (conversation: string, shareId: string): string =>
|
|
143
|
+
`${conversation}|device|${shareId}`,
|
|
144
|
+
} as const;
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Raise-only output-token backfill — the shared rule the cloud dispatch
|
|
3
|
+
* chain and the coreless daemon walker both apply so an alias / binding
|
|
4
|
+
* / pin that resolves to a concrete model gets its live vendor-reported
|
|
5
|
+
* ceiling instead of the client's conservative default for the
|
|
6
|
+
* unrecognised alias id.
|
|
7
|
+
*
|
|
8
|
+
* Lives in `@openllmsh/sdk/wire` (not `@openllm/core`) so the daemon can
|
|
9
|
+
* import it. Same gating as the historical `dispatch-chain` helper:
|
|
10
|
+
* concrete-id requests, unknown ceilings, and already-generous client
|
|
11
|
+
* values all no-op.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
export type TMaxTokensBackfillArgs = {
|
|
15
|
+
readonly requestedModel: string;
|
|
16
|
+
readonly getOutputTokenLimit: (modelId: string) => number | null;
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* When a hop's catalog id differs from the inbound requested id AND the
|
|
21
|
+
* resolved model's native output ceiling is known AND it exceeds the
|
|
22
|
+
* client's value, return that ceiling. Otherwise `null` (leave the
|
|
23
|
+
* client's cap untouched).
|
|
24
|
+
*/
|
|
25
|
+
export const backfilledMaxTokens = (
|
|
26
|
+
modelId: string,
|
|
27
|
+
clientMaxTokens: number | undefined,
|
|
28
|
+
args: TMaxTokensBackfillArgs,
|
|
29
|
+
): number | null => {
|
|
30
|
+
if (modelId === args.requestedModel) return null;
|
|
31
|
+
const ceiling = args.getOutputTokenLimit(modelId);
|
|
32
|
+
if (ceiling === null) return null;
|
|
33
|
+
if (clientMaxTokens !== undefined && clientMaxTokens >= ceiling) return null;
|
|
34
|
+
return ceiling;
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
const finiteNumber = (value: unknown): number | undefined =>
|
|
38
|
+
typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Effective client output cap on an already-built upstream body.
|
|
42
|
+
* The smallest present sibling wins: a leftover 32k `max_tokens` must
|
|
43
|
+
* still trigger raise-only backfill even when `max_completion_tokens`
|
|
44
|
+
* is already at the ceiling (first-present would miss it and leave
|
|
45
|
+
* the smaller field to shadow upstream).
|
|
46
|
+
*/
|
|
47
|
+
export const clientOutputCapFromBody = (body: unknown): number | undefined => {
|
|
48
|
+
if (body === null || typeof body !== "object") return undefined;
|
|
49
|
+
const rec = body as Record<string, unknown>;
|
|
50
|
+
const caps: number[] = [];
|
|
51
|
+
for (const key of [
|
|
52
|
+
"max_output_tokens",
|
|
53
|
+
"max_completion_tokens",
|
|
54
|
+
"max_tokens",
|
|
55
|
+
] as const) {
|
|
56
|
+
const value = finiteNumber(rec[key]);
|
|
57
|
+
if (value !== undefined) caps.push(value);
|
|
58
|
+
}
|
|
59
|
+
if (caps.length === 0) return undefined;
|
|
60
|
+
return Math.min(...caps);
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
const inboundRequestedModel = (rawBody: unknown): string | null => {
|
|
64
|
+
if (rawBody === null || typeof rawBody !== "object") return null;
|
|
65
|
+
const model = (rawBody as { readonly model?: unknown }).model;
|
|
66
|
+
return typeof model === "string" && model.length > 0 ? model : null;
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Raise whichever output-cap fields the built body already carries so a
|
|
71
|
+
* leftover smaller sibling cannot shadow the backfill. Codex Responses
|
|
72
|
+
* bodies omit the field on purpose (`max_output_tokens` 400s) — do not
|
|
73
|
+
* invent it. When no cap field is present on an openai / anthropic /
|
|
74
|
+
* non-Codex Responses body, write the field that wire honors.
|
|
75
|
+
*/
|
|
76
|
+
export const raiseOutputCapFields = (
|
|
77
|
+
body: unknown,
|
|
78
|
+
raised: number,
|
|
79
|
+
wire: "anthropic" | "chatgpt" | "openai",
|
|
80
|
+
// `false` = non-Codex Responses (Grok). Undefined / true = Codex.
|
|
81
|
+
codexInstructions?: boolean,
|
|
82
|
+
): unknown => {
|
|
83
|
+
if (body === null || typeof body !== "object" || Array.isArray(body)) {
|
|
84
|
+
return body;
|
|
85
|
+
}
|
|
86
|
+
const rec = body as Record<string, unknown>;
|
|
87
|
+
const hasOutput = finiteNumber(rec.max_output_tokens) !== undefined;
|
|
88
|
+
const hasCompletion = finiteNumber(rec.max_completion_tokens) !== undefined;
|
|
89
|
+
const hasTokens = finiteNumber(rec.max_tokens) !== undefined;
|
|
90
|
+
if (!hasOutput && !hasCompletion && !hasTokens) {
|
|
91
|
+
if (wire === "chatgpt" && codexInstructions !== false) return body;
|
|
92
|
+
if (wire === "chatgpt") return { ...rec, max_output_tokens: raised };
|
|
93
|
+
if (wire === "anthropic") return { ...rec, max_tokens: raised };
|
|
94
|
+
return { ...rec, max_completion_tokens: raised };
|
|
95
|
+
}
|
|
96
|
+
const raiseField = (existing: unknown): number => {
|
|
97
|
+
const current = finiteNumber(existing);
|
|
98
|
+
return current === undefined ? raised : Math.max(current, raised);
|
|
99
|
+
};
|
|
100
|
+
// Per-field raise-only: a sibling already above the ceiling stays;
|
|
101
|
+
// a leftover smaller sibling is lifted. Writing every field to
|
|
102
|
+
// `raised` would lower a generous cap.
|
|
103
|
+
return {
|
|
104
|
+
...rec,
|
|
105
|
+
...(hasOutput
|
|
106
|
+
? { max_output_tokens: raiseField(rec.max_output_tokens) }
|
|
107
|
+
: {}),
|
|
108
|
+
...(hasCompletion
|
|
109
|
+
? { max_completion_tokens: raiseField(rec.max_completion_tokens) }
|
|
110
|
+
: {}),
|
|
111
|
+
...(hasTokens ? { max_tokens: raiseField(rec.max_tokens) } : {}),
|
|
112
|
+
};
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Apply raise-only backfill to a built upstream body. `null` get-limit
|
|
117
|
+
* or a no-op rule returns `body` unchanged.
|
|
118
|
+
*/
|
|
119
|
+
export const applyOutputTokenBackfill = (
|
|
120
|
+
body: unknown,
|
|
121
|
+
args: {
|
|
122
|
+
readonly hopModelId: string;
|
|
123
|
+
readonly rawBody: unknown;
|
|
124
|
+
readonly getOutputTokenLimit: (modelId: string) => number | null;
|
|
125
|
+
readonly wire: "anthropic" | "chatgpt" | "openai";
|
|
126
|
+
readonly codexInstructions?: boolean;
|
|
127
|
+
},
|
|
128
|
+
): unknown => {
|
|
129
|
+
const requestedModel = inboundRequestedModel(args.rawBody) ?? args.hopModelId;
|
|
130
|
+
const raised = backfilledMaxTokens(
|
|
131
|
+
args.hopModelId,
|
|
132
|
+
clientOutputCapFromBody(body),
|
|
133
|
+
{
|
|
134
|
+
requestedModel,
|
|
135
|
+
getOutputTokenLimit: args.getOutputTokenLimit,
|
|
136
|
+
},
|
|
137
|
+
);
|
|
138
|
+
return raised === null
|
|
139
|
+
? body
|
|
140
|
+
: raiseOutputCapFields(body, raised, args.wire, args.codexInstructions);
|
|
141
|
+
};
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import type { TModelCaps } from "@openllmsh/sdk/protocol";
|
|
2
|
+
|
|
3
|
+
export type { TModelCaps };
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Catalog-declared, model-family request constraints. These run after wire
|
|
7
|
+
* encoding so the resolved model card can make the final provider-specific
|
|
8
|
+
* adjustment without teaching a wire format about model names.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
const otherMaxTokensField = (
|
|
12
|
+
field: NonNullable<TModelCaps["maxTokensField"]>,
|
|
13
|
+
): "max_tokens" | "max_completion_tokens" =>
|
|
14
|
+
field === "max_tokens" ? "max_completion_tokens" : "max_tokens";
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Apply the resolved model card's final outbound-body constraints without
|
|
18
|
+
* mutating the wire encoder's result.
|
|
19
|
+
*/
|
|
20
|
+
export const applyModelCaps = (
|
|
21
|
+
body: Record<string, unknown>,
|
|
22
|
+
caps: TModelCaps | undefined,
|
|
23
|
+
): Record<string, unknown> => {
|
|
24
|
+
if (caps === undefined) return body;
|
|
25
|
+
|
|
26
|
+
const next = { ...body };
|
|
27
|
+
for (const key of caps.deniedParams ?? []) {
|
|
28
|
+
delete next[key];
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
if (caps.maxTokensField !== undefined) {
|
|
32
|
+
const otherField = otherMaxTokensField(caps.maxTokensField);
|
|
33
|
+
if (otherField in next) {
|
|
34
|
+
if (!(caps.maxTokensField in next)) {
|
|
35
|
+
next[caps.maxTokensField] = next[otherField];
|
|
36
|
+
}
|
|
37
|
+
delete next[otherField];
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
if (caps.temperature === "only-1" && next.temperature !== 1) {
|
|
42
|
+
delete next.temperature;
|
|
43
|
+
} else if (caps.temperature === "clamp-0-1") {
|
|
44
|
+
const temperature = next.temperature;
|
|
45
|
+
if (typeof temperature === "number") {
|
|
46
|
+
next.temperature = Math.min(1, Math.max(0, temperature));
|
|
47
|
+
}
|
|
48
|
+
} else if (caps.temperature === "drop") {
|
|
49
|
+
delete next.temperature;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
return next;
|
|
53
|
+
};
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
TChatCompletionRequest,
|
|
3
|
+
TModelSettings,
|
|
4
|
+
} from "@openllmsh/sdk/protocol";
|
|
5
|
+
import { applyReasoningEffortFloor } from "@openllmsh/sdk/protocol";
|
|
6
|
+
|
|
7
|
+
/** Apply to a fresh hop-local canonical view, never the shared inbound request. */
|
|
8
|
+
export const applyModelSettings = (
|
|
9
|
+
request: TChatCompletionRequest,
|
|
10
|
+
settings: TModelSettings | undefined,
|
|
11
|
+
): TChatCompletionRequest => {
|
|
12
|
+
let next = request;
|
|
13
|
+
if (request.service_tier == null && settings?.service_tier !== undefined) {
|
|
14
|
+
next = { ...next, service_tier: settings.service_tier };
|
|
15
|
+
}
|
|
16
|
+
const effort = applyReasoningEffortFloor(
|
|
17
|
+
next.reasoning_effort,
|
|
18
|
+
settings?.min_reasoning_effort,
|
|
19
|
+
);
|
|
20
|
+
if (effort !== next.reasoning_effort) {
|
|
21
|
+
next = { ...next, reasoning_effort: effort };
|
|
22
|
+
}
|
|
23
|
+
return next;
|
|
24
|
+
};
|