@caupulican/pi-adaptative 0.81.26 → 0.81.29

package/dist/core/models/perf-profile.js

@@ -1,6 +1,8 @@
 import { createAssistantMessageEventStream, } from "@caupulican/pi-ai";
 const PERF_EWMA_ALPHA = 0.3;
 const CHARS_PER_TOKEN_ESTIMATE = 4;
+const DEFERRED_HEADERS_MAX_GAP_MS = 100;
+const DEFERRED_HEADERS_MIN_REQUEST_MS = 1_000;
 export const DEFAULT_ADAPTIVE_STREAM_IDLE_CEILING_MS = 30 * 60 * 1000;
 export function isModelPerfProfile(value) {
     if (typeof value !== "object" || value === null || Array.isArray(value))
@@ -88,61 +90,121 @@ export function withModelPerfProfile(streamFn, recorder) {
         let responseHeadersAtMs;
         let firstTokenAtMs;
         const originalOnResponse = streamOptions?.onResponse;
-        const inner = await streamFn(model, context, {
-            ...streamOptions,
-            onResponse: async (response, responseModel) => {
-                responseHeadersAtMs ??= nowMs();
-                await originalOnResponse?.(response, responseModel);
-            },
-        });
         const outer = createAssistantMessageEventStream();
+        let latest = emptyAssistantMessage(model);
+        let inner;
+        try {
+            inner = await streamFn(model, context, {
+                ...streamOptions,
+                onResponse: async (response, responseModel) => {
+                    responseHeadersAtMs ??= nowMs();
+                    await originalOnResponse?.(response, responseModel);
+                },
+            });
+        }
+        catch (error) {
+            outer.push(perfStreamFailure(latest, streamOptions?.signal?.aborted === true, error));
+            return outer;
+        }
         void (async () => {
             let terminalPushed = false;
-            for await (const event of inner) {
-                if (firstTokenAtMs === undefined && isFirstTokenEvent(event)) {
-                    firstTokenAtMs = nowMs();
+            try {
+                for await (const event of inner) {
+                    latest = assistantMessageFromEvent(event);
+                    if (firstTokenAtMs === undefined && isFirstTokenEvent(event)) {
+                        firstTokenAtMs = nowMs();
+                    }
+                    if (event.type === "done") {
+                        terminalPushed = true;
+                        recordSuccessfulStreamSample({
+                            completionTokens: event.message.usage.output,
+                            doneAtMs: nowMs(),
+                            firstTokenAtMs,
+                            model,
+                            nowIso,
+                            promptTokens: event.message.usage.input + event.message.usage.cacheRead + event.message.usage.cacheWrite,
+                            recorder,
+                            requestStartedAtMs,
+                            responseHeadersAtMs,
+                        });
+                    }
+                    else if (event.type === "error") {
+                        terminalPushed = true;
+                    }
+                    outer.push(event);
+                    if (terminalPushed)
+                        return;
                 }
-                if (event.type === "done") {
-                    terminalPushed = true;
-                    recordSuccessfulStreamSample({
-                        completionTokens: event.message.usage.output,
-                        doneAtMs: nowMs(),
-                        firstTokenAtMs,
-                        model,
-                        nowIso,
-                        promptTokens: event.message.usage.input,
-                        recorder,
-                        requestStartedAtMs,
-                        responseHeadersAtMs,
-                    });
-                }
-                else if (event.type === "error") {
-                    terminalPushed = true;
-                }
-                outer.push(event);
-                if (terminalPushed)
-                    return;
+                outer.push(perfStreamFailure(latest, streamOptions?.signal?.aborted === true));
+            }
+            catch (error) {
+                outer.push(perfStreamFailure(latest, streamOptions?.signal?.aborted === true, error));
             }
-            outer.end();
         })();
         return outer;
     };
 }
+function assistantMessageFromEvent(event) {
+    if (event.type === "done")
+        return event.message;
+    if (event.type === "error")
+        return event.error;
+    return event.partial;
+}
+function emptyAssistantMessage(model) {
+    return {
+        role: "assistant",
+        content: [],
+        api: model.api,
+        provider: model.provider,
+        model: model.id,
+        usage: {
+            input: 0,
+            output: 0,
+            cacheRead: 0,
+            cacheWrite: 0,
+            totalTokens: 0,
+            cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
+        },
+        stopReason: "stop",
+        timestamp: Date.now(),
+    };
+}
+function perfStreamFailure(latest, aborted, error) {
+    const stopReason = aborted ? "aborted" : "error";
+    const detail = error instanceof Error ? `: ${error.message}` : error === undefined ? "" : `: ${String(error)}`;
+    return {
+        type: "error",
+        reason: stopReason,
+        error: {
+            ...latest,
+            stopReason,
+            errorMessage: aborted
+                ? `stream aborted before terminal event${detail}`
+                : `stream ended without terminal event${detail}`,
+        },
+    };
+}
 function recordSuccessfulStreamSample(input) {
     if (input.firstTokenAtMs === undefined)
         return;
     const modelKey = input.recorder.modelKey(input.model);
     if (!modelKey)
         return;
+    const requestToFirstTokenMs = input.firstTokenAtMs - input.requestStartedAtMs;
     const sample = {
         promptTokens: input.promptTokens,
         completionTokens: input.completionTokens,
-        requestToFirstTokenMs: input.firstTokenAtMs - input.requestStartedAtMs,
+        requestToFirstTokenMs,
         firstTokenToDoneMs: input.doneAtMs - input.firstTokenAtMs,
         at: input.nowIso(),
     };
     if (input.responseHeadersAtMs !== undefined) {
-        sample.headersToFirstTokenMs = input.firstTokenAtMs - input.responseHeadersAtMs;
+        const headersToFirstTokenMs = input.firstTokenAtMs - input.responseHeadersAtMs;
+        const headersWereEffectivelyDeferred = requestToFirstTokenMs >= DEFERRED_HEADERS_MIN_REQUEST_MS &&
+            headersToFirstTokenMs <= DEFERRED_HEADERS_MAX_GAP_MS;
+        if (!headersWereEffectivelyDeferred)
+            sample.headersToFirstTokenMs = headersToFirstTokenMs;
     }
     if (!hasUsableModelPerfSample(sample))
         return;

package/dist/core/models/perf-profile.js.map

@@ -1 +1 @@
-{"version":3,"file":"perf-profile.js","sourceRoot":"","sources":["../../../src/core/models/perf-profile.ts"],"names":[],"mappings":"AACA,OAAO,EAIN,iCAAiC,GAGjC,MAAM,mBAAmB,CAAC;AAE3B,MAAM,eAAe,GAAG,GAAG,CAAC;AAC5B,MAAM,wBAAwB,GAAG,CAAC,CAAC;AACnC,MAAM,CAAC,MAAM,uCAAuC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAmCtE,MAAM,UAAU,kBAAkB,CAAC,KAAc,EAA6B;IAC7E,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACtF,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;IAC/B,OAAO,CACN,CAAC,MAAM,CAAC,sBAAsB,KAAK,SAAS,IAAI,sBAAsB,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;QACtG,CAAC,MAAM,CAAC,qBAAqB,KAAK,SAAS,IAAI,sBAAsB,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;QACpG,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,sBAAsB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACtE,OAAO,OAAO,KAAK,QAAQ;QAC3B,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC;QACzB,OAAO,IAAI,CAAC;QACZ,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ,CACpC,CAAC;AAAA,CACF;AAED,MAAM,UAAU,wBAAwB,CAAC,MAAuB,EAAW;IAC1E,OAAO,CACN,qBAAqB,CAAC,MAAM,CAAC,KAAK,SAAS;QAC3C,oBAAoB,CAAC,MAAM,CAAC,KAAK,SAAS;QAC1C,sBAAsB,CAAC,MAAM,CAAC,MAAM,CAAC,CACrC,CAAC;AAAA,CACF;AAED,MAAM,UAAU,sBAAsB,CACrC,OAAqC,EACrC,MAAuB,EACvB,EAAE,GAAW,MAAM,CAAC,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EACnB;IAC/B,MAAM,WAAW,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC;IAClD,MAAM,UAAU,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,sBAAsB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACjF,IAAI,WAAW,KAAK,SAAS,IAAI,UAAU,KAAK,SAAS,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,OAAO,CAAC;IAElG,MAAM,eAAe,GAAG,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC;IAC9C,OAAO;QACN,GAAG,CAAC,WAAW,KAAK,SAAS,IAAI;YAChC,sBAAsB,EAAE,UAAU,CAAC,OAAO,EAAE,sBAAsB,EAAE,WAAW,EAAE,eAAe,CAAC;SACjG,CAAC;QACF,GAAG,CAAC,WAAW,KAAK,SAAS,IAAI,OAAO,EAAE,sBAAsB,KAAK,SAAS;YAC7E,CAAC,CAAC,EAAE,sBAAsB,EAAE,OAAO,CAAC,sBAAsB,EAAE;YAC5D,CAAC,CAAC,EAAE,CAAC;QACN,GAAG,CAAC,UAAU,KAAK,SAAS,IAAI;YAC/B,qBAAqB,EAAE,UAAU,CAAC,OAAO,EAAE,qBAAqB,EAAE,UAAU,EAAE,eAAe,CAAC;SAC9F,CAAC;QACF,GAAG,CAAC,UAAU,KAAK,SAAS,IAAI,OAAO,EAAE,qBAAqB,KAAK,SAAS;YAC3E,CAAC,CAAC,EAAE,qBAAqB,EAAE,OAAO,CAAC,qBAAqB,EAAE;YAC1D,CAAC,CAAC,EAAE,CAAC;QACN,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,CAAC,EAAE,CAAC;QAC7F,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5F,OAAO,EAAE,eAAe,GAAG,CAAC;QAC5B,SAAS,EAAE,EAAE;KACb,CAAC;AAAA,CACF;AAED,MAAM,UAAU,gCAAgC,CAAC,KAA8B,EAA8B;IAC5G,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;IAC9B,MAAM,MAAM,GAA+B,EAAE,CAAC;IAC9C,MAAM,qBAAqB,GAAG,KAAK,CAAC,UAAU;QAC7C,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC;QACxD,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC;IACxB,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,CAAC,SAAS,IAAI,uCAAuC,CAAC,CAAC;IACpH,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,KAAK,CAAC,SAAS,IAAI,uCAAuC,CAAC,CAAC;IAErH,MAAM,iBAAiB,GAAG,0BAA0B,CAAC,OAAO,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAClF,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;QACrC,MAAM,WAAW,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,cAAc,EAAE,iBAAiB,CAAC,CAAC;QAC7F,IAAI,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW;YAAE,MAAM,CAAC,WAAW,GAAG,WAAW,CAAC;IAC5E,CAAC;IAED,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;QACtB,MAAM,qBAAqB,GAC1B,iBAAiB,KAAK,SAAS,CAAC,CAAC,CAAC,iBAAiB,GAAG,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC;QAChG,MAAM,SAAS,GACd,qBAAqB,KAAK,SAAS;YAClC,CAAC,CAAC,aAAa,CAAC,qBAAqB,EAAE,gBAAgB,EAAE,qBAAqB,CAAC;YAC/E,CAAC,CAAC,qBAAqB,CAAC;QAC1B,IAAI,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS;YAAE,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC;IACpE,CAAC;IAED,OAAO,MAAM,CAAC;AAAA,CACd;AAED,MAAM,UAAU,2BAA2B,CAAC,OAAgB,EAAU;IACrE,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC;QACjC,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;KACpB,CAAC,CAAC;IACH,IAAI,CAAC,UAAU;QAAE,OAAO,CAAC,CAAC;IAC1B,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,wBAAwB,CAAC,CAAC;AAAA,CAC/D;AAED,MAAM,UAAU,oBAAoB,CAAC,QAAkB,EAAE,QAAwC,EAAY;IAC5G,OAAO,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,EAAE,CAAC;QAC/C,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC;QACzC,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;QACnE,MAAM,kBAAkB,GAAG,KAAK,EAAE,CAAC;QACnC,IAAI,mBAAuC,CAAC;QAC5C,IAAI,cAAkC,CAAC;QACvC,MAAM,kBAAkB,GAAG,aAAa,EAAE,UAAU,CAAC;QACrD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE;YAC5C,GAAG,aAAa;YAChB,UAAU,EAAE,KAAK,EAAE,QAA0B,EAAE,aAAyB,EAAE,EAAE,CAAC;gBAC5E,mBAAmB,KAAK,KAAK,EAAE,CAAC;gBAChC,MAAM,kBAAkB,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;YAAA,CACpD;SACD,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,iCAAiC,EAAE,CAAC;QAElD,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;YACjB,IAAI,cAAc,GAAG,KAAK,CAAC;YAC3B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;gBACjC,IAAI,cAAc,KAAK,SAAS,IAAI,iBAAiB,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC9D,cAAc,GAAG,KAAK,EAAE,CAAC;gBAC1B,CAAC;gBACD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;oBAC3B,cAAc,GAAG,IAAI,CAAC;oBACtB,4BAA4B,CAAC;wBAC5B,gBAAgB,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM;wBAC5C,QAAQ,EAAE,KAAK,EAAE;wBACjB,cAAc;wBACd,KAAK;wBACL,MAAM;wBACN,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK;wBACvC,QAAQ;wBACR,kBAAkB;wBAClB,mBAAmB;qBACnB,CAAC,CAAC;gBACJ,CAAC;qBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;oBACnC,cAAc,GAAG,IAAI,CAAC;gBACvB,CAAC;gBACD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAClB,IAAI,cAAc;oBAAE,OAAO;YAC5B,CAAC;YACD,KAAK,CAAC,GAAG,EAAE,CAAC;QAAA,CACZ,CAAC,EAAE,CAAC;QAEL,OAAO,KAAK,CAAC;IAAA,CACb,CAAC;AAAA,CACF;AAED,SAAS,4BAA4B,CAAC,KAUrC,EAAQ;IACR,IAAI,KAAK,CAAC,cAAc,KAAK,SAAS;QAAE,OAAO;IAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACtD,IAAI,CAAC,QAAQ;QAAE,OAAO;IACtB,MAAM,MAAM,GAAoB;QAC/B,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;QACxC,qBAAqB,EAAE,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,kBAAkB;QACtE,kBAAkB,EAAE,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,cAAc;QACzD,EAAE,EAAE,KAAK,CAAC,MAAM,EAAE;KAClB,CAAC;IACF,IAAI,KAAK,CAAC,mBAAmB,KAAK,SAAS,EAAE,CAAC;QAC7C,MAAM,CAAC,qBAAqB,GAAG,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,mBAAmB,CAAC;IACjF,CAAC;IACD,IAAI,CAAC,wBAAwB,CAAC,MAAM,CAAC;QAAE,OAAO;IAC9C,IAAI,CAAC;QACJ,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACR,gDAAgD;IACjD,CAAC;AAAA,CACD;AAED,SAAS,iBAAiB,CAAC,KAA4B,EAAW;IACjE,OAAO,CACN,CAAC,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,gBAAgB,IAAI,KAAK,CAAC,IAAI,KAAK,gBAAgB,CAAC;QACnG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CACtB,CAAC;AAAA,CACF;AAED,SAAS,0BAA0B,CAAC,OAAqC,EAAE,YAAoB,EAAsB;IACpH,MAAM,WAAW,GAAG,OAAO,EAAE,sBAAsB,CAAC;IACpD,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,YAAY,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IAC3F,MAAM,iBAAiB,GAAG,CAAC,YAAY,GAAG,WAAW,CAAC,GAAG,IAAI,CAAC;IAC9D,OAAO,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,iBAAiB,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,SAAS,CAAC;AAAA,CACnG;AAED,SAAS,aAAa,CAAC,SAAiB,EAAE,SAAiB,EAAE,UAAkB,EAAU;IACxF,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;AAAA,CAC3E;AAED,SAAS,qBAAqB,CAAC,MAAuB,EAAsB;IAC3E,MAAM,UAAU,GAAG,sBAAsB,CAAC,MAAM,CAAC,qBAAqB,CAAC;QACtE,CAAC,CAAC,MAAM,CAAC,qBAAqB;QAC9B,CAAC,CAAC,MAAM,CAAC,qBAAqB,CAAC;IAChC,OAAO,eAAe,CAAC,MAAM,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;AAAA,CACxD;AAED,SAAS,oBAAoB,CAAC,MAAuB,EAAsB;IAC1E,OAAO,eAAe,CAAC,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAAC,kBAAkB,CAAC,CAAC;AAAA,CAC3E;AAED,SAAS,eAAe,CAAC,MAA0B,EAAE,UAA8B,EAAsB;IACxG,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,CAAC,UAAU,CAAC;QAAE,OAAO,SAAS,CAAC;IAC7F,OAAO,MAAM,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;AAAA,CACpC;AAED,SAAS,UAAU,CAAC,QAA4B,EAAE,IAAY,EAAE,eAAuB,EAAU;IAChG,IAAI,QAAQ,KAAK,SAAS,IAAI,eAAe,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACjE,OAAO,QAAQ,GAAG,CAAC,CAAC,GAAG,eAAe,CAAC,GAAG,IAAI,GAAG,eAAe,CAAC;AAAA,CACjE;AAED,SAAS,sBAAsB,CAAC,KAAc,EAAmB;IAChE,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC;AAAA,CACxE","sourcesContent":["import type { StreamFn, StreamIdleOptions } from \"@caupulican/pi-agent-core\";\nimport {\n\ttype Api,\n\ttype AssistantMessageEvent,\n\ttype Context,\n\tcreateAssistantMessageEventStream,\n\ttype Model,\n\ttype ProviderResponse,\n} from \"@caupulican/pi-ai\";\n\nconst PERF_EWMA_ALPHA = 0.3;\nconst CHARS_PER_TOKEN_ESTIMATE = 4;\nexport const DEFAULT_ADAPTIVE_STREAM_IDLE_CEILING_MS = 30 * 60 * 1000;\n\nexport interface ModelPerfProfile {\n\tprefillTokensPerSecond?: number;\n\tdecodeTokensPerSecond?: number;\n\tloadMs?: number;\n\tsamples: number;\n\tupdatedAt: string;\n}\n\nexport interface ModelPerfSample {\n\tpromptTokens?: number;\n\tcompletionTokens?: number;\n\theadersToFirstTokenMs?: number;\n\trequestToFirstTokenMs?: number;\n\tfirstTokenToDoneMs?: number;\n\tloadMs?: number;\n\tat?: string;\n}\n\nexport interface AdaptiveStreamIdleInput {\n\tbase: StreamIdleOptions;\n\tprofile?: ModelPerfProfile;\n\tpromptTokens: number;\n\tceilingMs?: number;\n\tlocalClass?: boolean;\n}\n\nexport interface ModelPerfProfileStreamRecorder {\n\tmodelKey: (model: Model<Api>) => string | undefined;\n\trecordSample: (modelKey: string, sample: ModelPerfSample) => void;\n\tnowMs?: () => number;\n\tnowIso?: () => string;\n}\n\nexport function isModelPerfProfile(value: unknown): value is ModelPerfProfile {\n\tif (typeof value !== \"object\" || value === null || Array.isArray(value)) return false;\n\tconst record = value as Record<string, unknown>;\n\tconst samples = record.samples;\n\treturn (\n\t\t(record.prefillTokensPerSecond === undefined || isPositiveFiniteNumber(record.prefillTokensPerSecond)) &&\n\t\t(record.decodeTokensPerSecond === undefined || isPositiveFiniteNumber(record.decodeTokensPerSecond)) &&\n\t\t(record.loadMs === undefined || isPositiveFiniteNumber(record.loadMs)) &&\n\t\ttypeof samples === \"number\" &&\n\t\tNumber.isInteger(samples) &&\n\t\tsamples >= 0 &&\n\t\ttypeof record.updatedAt === \"string\"\n\t);\n}\n\nexport function hasUsableModelPerfSample(sample: ModelPerfSample): boolean {\n\treturn (\n\t\tprefillRateFromSample(sample) !== undefined ||\n\t\tdecodeRateFromSample(sample) !== undefined ||\n\t\tisPositiveFiniteNumber(sample.loadMs)\n\t);\n}\n\nexport function updateModelPerfProfile(\n\tcurrent: ModelPerfProfile | undefined,\n\tsample: ModelPerfSample,\n\tat: string = sample.at ?? new Date().toISOString(),\n): ModelPerfProfile | undefined {\n\tconst prefillRate = prefillRateFromSample(sample);\n\tconst decodeRate = decodeRateFromSample(sample);\n\tconst loadMs = isPositiveFiniteNumber(sample.loadMs) ? sample.loadMs : undefined;\n\tif (prefillRate === undefined && decodeRate === undefined && loadMs === undefined) return current;\n\n\tconst previousSamples = current?.samples ?? 0;\n\treturn {\n\t\t...(prefillRate !== undefined && {\n\t\t\tprefillTokensPerSecond: updateEwma(current?.prefillTokensPerSecond, prefillRate, previousSamples),\n\t\t}),\n\t\t...(prefillRate === undefined && current?.prefillTokensPerSecond !== undefined\n\t\t\t? { prefillTokensPerSecond: current.prefillTokensPerSecond }\n\t\t\t: {}),\n\t\t...(decodeRate !== undefined && {\n\t\t\tdecodeTokensPerSecond: updateEwma(current?.decodeTokensPerSecond, decodeRate, previousSamples),\n\t\t}),\n\t\t...(decodeRate === undefined && current?.decodeTokensPerSecond !== undefined\n\t\t\t? { decodeTokensPerSecond: current.decodeTokensPerSecond }\n\t\t\t: {}),\n\t\t...(loadMs !== undefined && { loadMs: updateEwma(current?.loadMs, loadMs, previousSamples) }),\n\t\t...(loadMs === undefined && current?.loadMs !== undefined ? { loadMs: current.loadMs } : {}),\n\t\tsamples: previousSamples + 1,\n\t\tupdatedAt: at,\n\t};\n}\n\nexport function resolveAdaptiveStreamIdleOptions(input: AdaptiveStreamIdleInput): Partial<StreamIdleOptions> {\n\tconst profile = input.profile;\n\tconst result: Partial<StreamIdleOptions> = {};\n\tconst localConnectDefaultMs = input.localClass\n\t\t? Math.max(input.base.connectMs, input.base.quietIdleMs)\n\t\t: input.base.connectMs;\n\tconst quietCeilingMs = Math.max(input.base.quietIdleMs, input.ceilingMs ?? DEFAULT_ADAPTIVE_STREAM_IDLE_CEILING_MS);\n\tconst connectCeilingMs = Math.max(localConnectDefaultMs, input.ceilingMs ?? DEFAULT_ADAPTIVE_STREAM_IDLE_CEILING_MS);\n\n\tconst expectedPrefillMs = expectedPrefillFromProfile(profile, input.promptTokens);\n\tif (expectedPrefillMs !== undefined) {\n\t\tconst quietIdleMs = adaptiveBound(input.base.quietIdleMs, quietCeilingMs, expectedPrefillMs);\n\t\tif (quietIdleMs > input.base.quietIdleMs) result.quietIdleMs = quietIdleMs;\n\t}\n\n\tif (input.localClass) {\n\t\tconst expectedConnectWorkMs =\n\t\t\texpectedPrefillMs !== undefined ? expectedPrefillMs + (profile?.loadMs ?? 0) : profile?.loadMs;\n\t\tconst connectMs =\n\t\t\texpectedConnectWorkMs !== undefined\n\t\t\t\t? adaptiveBound(localConnectDefaultMs, connectCeilingMs, expectedConnectWorkMs)\n\t\t\t\t: localConnectDefaultMs;\n\t\tif (connectMs > input.base.connectMs) result.connectMs = connectMs;\n\t}\n\n\treturn result;\n}\n\nexport function estimateContextPromptTokens(context: Context): number {\n\tconst serialized = JSON.stringify({\n\t\tsystemPrompt: context.systemPrompt,\n\t\tmessages: context.messages,\n\t\ttools: context.tools,\n\t});\n\tif (!serialized) return 0;\n\treturn Math.ceil(serialized.length / CHARS_PER_TOKEN_ESTIMATE);\n}\n\nexport function withModelPerfProfile(streamFn: StreamFn, recorder: ModelPerfProfileStreamRecorder): StreamFn {\n\treturn async (model, context, streamOptions) => {\n\t\tconst nowMs = recorder.nowMs ?? Date.now;\n\t\tconst nowIso = recorder.nowIso ?? (() => new Date().toISOString());\n\t\tconst requestStartedAtMs = nowMs();\n\t\tlet responseHeadersAtMs: number | undefined;\n\t\tlet firstTokenAtMs: number | undefined;\n\t\tconst originalOnResponse = streamOptions?.onResponse;\n\t\tconst inner = await streamFn(model, context, {\n\t\t\t...streamOptions,\n\t\t\tonResponse: async (response: ProviderResponse, responseModel: Model<Api>) => {\n\t\t\t\tresponseHeadersAtMs ??= nowMs();\n\t\t\t\tawait originalOnResponse?.(response, responseModel);\n\t\t\t},\n\t\t});\n\t\tconst outer = createAssistantMessageEventStream();\n\n\t\tvoid (async () => {\n\t\t\tlet terminalPushed = false;\n\t\t\tfor await (const event of inner) {\n\t\t\t\tif (firstTokenAtMs === undefined && isFirstTokenEvent(event)) {\n\t\t\t\t\tfirstTokenAtMs = nowMs();\n\t\t\t\t}\n\t\t\t\tif (event.type === \"done\") {\n\t\t\t\t\tterminalPushed = true;\n\t\t\t\t\trecordSuccessfulStreamSample({\n\t\t\t\t\t\tcompletionTokens: event.message.usage.output,\n\t\t\t\t\t\tdoneAtMs: nowMs(),\n\t\t\t\t\t\tfirstTokenAtMs,\n\t\t\t\t\t\tmodel,\n\t\t\t\t\t\tnowIso,\n\t\t\t\t\t\tpromptTokens: event.message.usage.input,\n\t\t\t\t\t\trecorder,\n\t\t\t\t\t\trequestStartedAtMs,\n\t\t\t\t\t\tresponseHeadersAtMs,\n\t\t\t\t\t});\n\t\t\t\t} else if (event.type === \"error\") {\n\t\t\t\t\tterminalPushed = true;\n\t\t\t\t}\n\t\t\t\touter.push(event);\n\t\t\t\tif (terminalPushed) return;\n\t\t\t}\n\t\t\touter.end();\n\t\t})();\n\n\t\treturn outer;\n\t};\n}\n\nfunction recordSuccessfulStreamSample(input: {\n\tmodel: Model<Api>;\n\trecorder: ModelPerfProfileStreamRecorder;\n\trequestStartedAtMs: number;\n\tresponseHeadersAtMs: number | undefined;\n\tfirstTokenAtMs: number | undefined;\n\tdoneAtMs: number;\n\tpromptTokens: number;\n\tcompletionTokens: number;\n\tnowIso: () => string;\n}): void {\n\tif (input.firstTokenAtMs === undefined) return;\n\tconst modelKey = input.recorder.modelKey(input.model);\n\tif (!modelKey) return;\n\tconst sample: ModelPerfSample = {\n\t\tpromptTokens: input.promptTokens,\n\t\tcompletionTokens: input.completionTokens,\n\t\trequestToFirstTokenMs: input.firstTokenAtMs - input.requestStartedAtMs,\n\t\tfirstTokenToDoneMs: input.doneAtMs - input.firstTokenAtMs,\n\t\tat: input.nowIso(),\n\t};\n\tif (input.responseHeadersAtMs !== undefined) {\n\t\tsample.headersToFirstTokenMs = input.firstTokenAtMs - input.responseHeadersAtMs;\n\t}\n\tif (!hasUsableModelPerfSample(sample)) return;\n\ttry {\n\t\tinput.recorder.recordSample(modelKey, sample);\n\t} catch {\n\t\t// Perf profiling must never fail the user turn.\n\t}\n}\n\nfunction isFirstTokenEvent(event: AssistantMessageEvent): boolean {\n\treturn (\n\t\t(event.type === \"text_delta\" || event.type === \"thinking_delta\" || event.type === \"toolcall_delta\") &&\n\t\tevent.delta.length > 0\n\t);\n}\n\nfunction expectedPrefillFromProfile(profile: ModelPerfProfile | undefined, promptTokens: number): number | undefined {\n\tconst prefillRate = profile?.prefillTokensPerSecond;\n\tif (!profile || profile.samples < 1 || !prefillRate || promptTokens <= 0) return undefined;\n\tconst expectedPrefillMs = (promptTokens / prefillRate) * 1000;\n\treturn Number.isFinite(expectedPrefillMs) && expectedPrefillMs > 0 ? expectedPrefillMs : undefined;\n}\n\nfunction adaptiveBound(defaultMs: number, ceilingMs: number, expectedMs: number): number {\n\treturn Math.min(Math.ceil(Math.max(defaultMs, expectedMs * 3)), ceilingMs);\n}\n\nfunction prefillRateFromSample(sample: ModelPerfSample): number | undefined {\n\tconst durationMs = isPositiveFiniteNumber(sample.headersToFirstTokenMs)\n\t\t? sample.headersToFirstTokenMs\n\t\t: sample.requestToFirstTokenMs;\n\treturn tokensPerSecond(sample.promptTokens, durationMs);\n}\n\nfunction decodeRateFromSample(sample: ModelPerfSample): number | undefined {\n\treturn tokensPerSecond(sample.completionTokens, sample.firstTokenToDoneMs);\n}\n\nfunction tokensPerSecond(tokens: number | undefined, durationMs: number | undefined): number | undefined {\n\tif (!isPositiveFiniteNumber(tokens) || !isPositiveFiniteNumber(durationMs)) return undefined;\n\treturn tokens / (durationMs / 1000);\n}\n\nfunction updateEwma(previous: number | undefined, next: number, previousSamples: number): number {\n\tif (previous === undefined || previousSamples === 0) return next;\n\treturn previous * (1 - PERF_EWMA_ALPHA) + next * PERF_EWMA_ALPHA;\n}\n\nfunction isPositiveFiniteNumber(value: unknown): value is number {\n\treturn typeof value === \"number\" && Number.isFinite(value) && value > 0;\n}\n"]}
+{"version":3,"file":"perf-profile.js","sourceRoot":"","sources":["../../../src/core/models/perf-profile.ts"],"names":[],"mappings":"AACA,OAAO,EAKN,iCAAiC,GAGjC,MAAM,mBAAmB,CAAC;AAE3B,MAAM,eAAe,GAAG,GAAG,CAAC;AAC5B,MAAM,wBAAwB,GAAG,CAAC,CAAC;AACnC,MAAM,2BAA2B,GAAG,GAAG,CAAC;AACxC,MAAM,+BAA+B,GAAG,KAAK,CAAC;AAC9C,MAAM,CAAC,MAAM,uCAAuC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAmCtE,MAAM,UAAU,kBAAkB,CAAC,KAAc,EAA6B;IAC7E,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACtF,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;IAC/B,OAAO,CACN,CAAC,MAAM,CAAC,sBAAsB,KAAK,SAAS,IAAI,sBAAsB,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;QACtG,CAAC,MAAM,CAAC,qBAAqB,KAAK,SAAS,IAAI,sBAAsB,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;QACpG,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,sBAAsB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACtE,OAAO,OAAO,KAAK,QAAQ;QAC3B,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC;QACzB,OAAO,IAAI,CAAC;QACZ,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ,CACpC,CAAC;AAAA,CACF;AAED,MAAM,UAAU,wBAAwB,CAAC,MAAuB,EAAW;IAC1E,OAAO,CACN,qBAAqB,CAAC,MAAM,CAAC,KAAK,SAAS;QAC3C,oBAAoB,CAAC,MAAM,CAAC,KAAK,SAAS;QAC1C,sBAAsB,CAAC,MAAM,CAAC,MAAM,CAAC,CACrC,CAAC;AAAA,CACF;AAED,MAAM,UAAU,sBAAsB,CACrC,OAAqC,EACrC,MAAuB,EACvB,EAAE,GAAW,MAAM,CAAC,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EACnB;IAC/B,MAAM,WAAW,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC;IAClD,MAAM,UAAU,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,sBAAsB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACjF,IAAI,WAAW,KAAK,SAAS,IAAI,UAAU,KAAK,SAAS,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,OAAO,CAAC;IAElG,MAAM,eAAe,GAAG,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC;IAC9C,OAAO;QACN,GAAG,CAAC,WAAW,KAAK,SAAS,IAAI;YAChC,sBAAsB,EAAE,UAAU,CAAC,OAAO,EAAE,sBAAsB,EAAE,WAAW,EAAE,eAAe,CAAC;SACjG,CAAC;QACF,GAAG,CAAC,WAAW,KAAK,SAAS,IAAI,OAAO,EAAE,sBAAsB,KAAK,SAAS;YAC7E,CAAC,CAAC,EAAE,sBAAsB,EAAE,OAAO,CAAC,sBAAsB,EAAE;YAC5D,CAAC,CAAC,EAAE,CAAC;QACN,GAAG,CAAC,UAAU,KAAK,SAAS,IAAI;YAC/B,qBAAqB,EAAE,UAAU,CAAC,OAAO,EAAE,qBAAqB,EAAE,UAAU,EAAE,eAAe,CAAC;SAC9F,CAAC;QACF,GAAG,CAAC,UAAU,KAAK,SAAS,IAAI,OAAO,EAAE,qBAAqB,KAAK,SAAS;YAC3E,CAAC,CAAC,EAAE,qBAAqB,EAAE,OAAO,CAAC,qBAAqB,EAAE;YAC1D,CAAC,CAAC,EAAE,CAAC;QACN,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,CAAC,EAAE,CAAC;QAC7F,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5F,OAAO,EAAE,eAAe,GAAG,CAAC;QAC5B,SAAS,EAAE,EAAE;KACb,CAAC;AAAA,CACF;AAED,MAAM,UAAU,gCAAgC,CAAC,KAA8B,EAA8B;IAC5G,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;IAC9B,MAAM,MAAM,GAA+B,EAAE,CAAC;IAC9C,MAAM,qBAAqB,GAAG,KAAK,CAAC,UAAU;QAC7C,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC;QACxD,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC;IACxB,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,CAAC,SAAS,IAAI,uCAAuC,CAAC,CAAC;IACpH,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,KAAK,CAAC,SAAS,IAAI,uCAAuC,CAAC,CAAC;IAErH,MAAM,iBAAiB,GAAG,0BAA0B,CAAC,OAAO,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAClF,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;QACrC,MAAM,WAAW,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,cAAc,EAAE,iBAAiB,CAAC,CAAC;QAC7F,IAAI,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW;YAAE,MAAM,CAAC,WAAW,GAAG,WAAW,CAAC;IAC5E,CAAC;IAED,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;QACtB,MAAM,qBAAqB,GAC1B,iBAAiB,KAAK,SAAS,CAAC,CAAC,CAAC,iBAAiB,GAAG,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC;QAChG,MAAM,SAAS,GACd,qBAAqB,KAAK,SAAS;YAClC,CAAC,CAAC,aAAa,CAAC,qBAAqB,EAAE,gBAAgB,EAAE,qBAAqB,CAAC;YAC/E,CAAC,CAAC,qBAAqB,CAAC;QAC1B,IAAI,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS;YAAE,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC;IACpE,CAAC;IAED,OAAO,MAAM,CAAC;AAAA,CACd;AAED,MAAM,UAAU,2BAA2B,CAAC,OAAgB,EAAU;IACrE,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC;QACjC,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;KACpB,CAAC,CAAC;IACH,IAAI,CAAC,UAAU;QAAE,OAAO,CAAC,CAAC;IAC1B,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,wBAAwB,CAAC,CAAC;AAAA,CAC/D;AAED,MAAM,UAAU,oBAAoB,CAAC,QAAkB,EAAE,QAAwC,EAAY;IAC5G,OAAO,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,EAAE,CAAC;QAC/C,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC;QACzC,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;QACnE,MAAM,kBAAkB,GAAG,KAAK,EAAE,CAAC;QACnC,IAAI,mBAAuC,CAAC;QAC5C,IAAI,cAAkC,CAAC;QACvC,MAAM,kBAAkB,GAAG,aAAa,EAAE,UAAU,CAAC;QACrD,MAAM,KAAK,GAAG,iCAAiC,EAAE,CAAC;QAClD,IAAI,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;QAC1C,IAAI,KAAoC,CAAC;QACzC,IAAI,CAAC;YACJ,KAAK,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE;gBACtC,GAAG,aAAa;gBAChB,UAAU,EAAE,KAAK,EAAE,QAA0B,EAAE,aAAyB,EAAE,EAAE,CAAC;oBAC5E,mBAAmB,KAAK,KAAK,EAAE,CAAC;oBAChC,MAAM,kBAAkB,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;gBAAA,CACpD;aACD,CAAC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,OAAO,KAAK,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;YACtF,OAAO,KAAK,CAAC;QACd,CAAC;QAED,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;YACjB,IAAI,cAAc,GAAG,KAAK,CAAC;YAC3B,IAAI,CAAC;gBACJ,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;oBACjC,MAAM,GAAG,yBAAyB,CAAC,KAAK,CAAC,CAAC;oBAC1C,IAAI,cAAc,KAAK,SAAS,IAAI,iBAAiB,CAAC,KAAK,CAAC,EAAE,CAAC;wBAC9D,cAAc,GAAG,KAAK,EAAE,CAAC;oBAC1B,CAAC;oBACD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;wBAC3B,cAAc,GAAG,IAAI,CAAC;wBACtB,4BAA4B,CAAC;4BAC5B,gBAAgB,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM;4BAC5C,QAAQ,EAAE,KAAK,EAAE;4BACjB,cAAc;4BACd,KAAK;4BACL,MAAM;4BACN,YAAY,EACX,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU;4BAC3F,QAAQ;4BACR,kBAAkB;4BAClB,mBAAmB;yBACnB,CAAC,CAAC;oBACJ,CAAC;yBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;wBACnC,cAAc,GAAG,IAAI,CAAC;oBACvB,CAAC;oBACD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBAClB,IAAI,cAAc;wBAAE,OAAO;gBAC5B,CAAC;gBACD,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC;YAChF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,OAAO,KAAK,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;YACvF,CAAC;QAAA,CACD,CAAC,EAAE,CAAC;QAEL,OAAO,KAAK,CAAC;IAAA,CACb,CAAC;AAAA,CACF;AAED,SAAS,yBAAyB,CAAC,KAA4B,EAAoB;IAClF,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,KAAK,CAAC,OAAO,CAAC;IAChD,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO;QAAE,OAAO,KAAK,CAAC,KAAK,CAAC;IAC/C,OAAO,KAAK,CAAC,OAAO,CAAC;AAAA,CACrB;AAED,SAAS,qBAAqB,CAAC,KAAiB,EAAoB;IACnE,OAAO;QACN,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,EAAE;QACX,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,KAAK,EAAE,KAAK,CAAC,EAAE;QACf,KAAK,EAAE;YACN,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,CAAC;YACT,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,CAAC;YACb,WAAW,EAAE,CAAC;YACd,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;SACpE;QACD,UAAU,EAAE,MAAM;QAClB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;KACrB,CAAC;AAAA,CACF;AAED,SAAS,iBAAiB,CAAC,MAAwB,EAAE,OAAgB,EAAE,KAAe,EAAyB;IAC9G,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;IACjD,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;IAC/G,OAAO;QACN,IAAI,EAAE,OAAO;QACb,MAAM,EAAE,UAAU;QAClB,KAAK,EAAE;YACN,GAAG,MAAM;YACT,UAAU;YACV,YAAY,EAAE,OAAO;gBACpB,CAAC,CAAC,uCAAuC,MAAM,EAAE;gBACjD,CAAC,CAAC,sCAAsC,MAAM,EAAE;SACjD;KACD,CAAC;AAAA,CACF;AAED,SAAS,4BAA4B,CAAC,KAUrC,EAAQ;IACR,IAAI,KAAK,CAAC,cAAc,KAAK,SAAS;QAAE,OAAO;IAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACtD,IAAI,CAAC,QAAQ;QAAE,OAAO;IACtB,MAAM,qBAAqB,GAAG,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,kBAAkB,CAAC;IAC9E,MAAM,MAAM,GAAoB;QAC/B,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;QACxC,qBAAqB;QACrB,kBAAkB,EAAE,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,cAAc;QACzD,EAAE,EAAE,KAAK,CAAC,MAAM,EAAE;KAClB,CAAC;IACF,IAAI,KAAK,CAAC,mBAAmB,KAAK,SAAS,EAAE,CAAC;QAC7C,MAAM,qBAAqB,GAAG,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,mBAAmB,CAAC;QAC/E,MAAM,8BAA8B,GACnC,qBAAqB,IAAI,+BAA+B;YACxD,qBAAqB,IAAI,2BAA2B,CAAC;QACtD,IAAI,CAAC,8BAA8B;YAAE,MAAM,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;IAC3F,CAAC;IACD,IAAI,CAAC,wBAAwB,CAAC,MAAM,CAAC;QAAE,OAAO;IAC9C,IAAI,CAAC;QACJ,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACR,gDAAgD;IACjD,CAAC;AAAA,CACD;AAED,SAAS,iBAAiB,CAAC,KAA4B,EAAW;IACjE,OAAO,CACN,CAAC,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,gBAAgB,IAAI,KAAK,CAAC,IAAI,KAAK,gBAAgB,CAAC;QACnG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CACtB,CAAC;AAAA,CACF;AAED,SAAS,0BAA0B,CAAC,OAAqC,EAAE,YAAoB,EAAsB;IACpH,MAAM,WAAW,GAAG,OAAO,EAAE,sBAAsB,CAAC;IACpD,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,YAAY,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IAC3F,MAAM,iBAAiB,GAAG,CAAC,YAAY,GAAG,WAAW,CAAC,GAAG,IAAI,CAAC;IAC9D,OAAO,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,iBAAiB,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,SAAS,CAAC;AAAA,CACnG;AAED,SAAS,aAAa,CAAC,SAAiB,EAAE,SAAiB,EAAE,UAAkB,EAAU;IACxF,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;AAAA,CAC3E;AAED,SAAS,qBAAqB,CAAC,MAAuB,EAAsB;IAC3E,MAAM,UAAU,GAAG,sBAAsB,CAAC,MAAM,CAAC,qBAAqB,CAAC;QACtE,CAAC,CAAC,MAAM,CAAC,qBAAqB;QAC9B,CAAC,CAAC,MAAM,CAAC,qBAAqB,CAAC;IAChC,OAAO,eAAe,CAAC,MAAM,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;AAAA,CACxD;AAED,SAAS,oBAAoB,CAAC,MAAuB,EAAsB;IAC1E,OAAO,eAAe,CAAC,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAAC,kBAAkB,CAAC,CAAC;AAAA,CAC3E;AAED,SAAS,eAAe,CAAC,MAA0B,EAAE,UAA8B,EAAsB;IACxG,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,CAAC,UAAU,CAAC;QAAE,OAAO,SAAS,CAAC;IAC7F,OAAO,MAAM,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;AAAA,CACpC;AAED,SAAS,UAAU,CAAC,QAA4B,EAAE,IAAY,EAAE,eAAuB,EAAU;IAChG,IAAI,QAAQ,KAAK,SAAS,IAAI,eAAe,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACjE,OAAO,QAAQ,GAAG,CAAC,CAAC,GAAG,eAAe,CAAC,GAAG,IAAI,GAAG,eAAe,CAAC;AAAA,CACjE;AAED,SAAS,sBAAsB,CAAC,KAAc,EAAmB;IAChE,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC;AAAA,CACxE","sourcesContent":["import type { StreamFn, StreamIdleOptions } from \"@caupulican/pi-agent-core\";\nimport {\n\ttype Api,\n\ttype AssistantMessage,\n\ttype AssistantMessageEvent,\n\ttype Context,\n\tcreateAssistantMessageEventStream,\n\ttype Model,\n\ttype ProviderResponse,\n} from \"@caupulican/pi-ai\";\n\nconst PERF_EWMA_ALPHA = 0.3;\nconst CHARS_PER_TOKEN_ESTIMATE = 4;\nconst DEFERRED_HEADERS_MAX_GAP_MS = 100;\nconst DEFERRED_HEADERS_MIN_REQUEST_MS = 1_000;\nexport const DEFAULT_ADAPTIVE_STREAM_IDLE_CEILING_MS = 30 * 60 * 1000;\n\nexport interface ModelPerfProfile {\n\tprefillTokensPerSecond?: number;\n\tdecodeTokensPerSecond?: number;\n\tloadMs?: number;\n\tsamples: number;\n\tupdatedAt: string;\n}\n\nexport interface ModelPerfSample {\n\tpromptTokens?: number;\n\tcompletionTokens?: number;\n\theadersToFirstTokenMs?: number;\n\trequestToFirstTokenMs?: number;\n\tfirstTokenToDoneMs?: number;\n\tloadMs?: number;\n\tat?: string;\n}\n\nexport interface AdaptiveStreamIdleInput {\n\tbase: StreamIdleOptions;\n\tprofile?: ModelPerfProfile;\n\tpromptTokens: number;\n\tceilingMs?: number;\n\tlocalClass?: boolean;\n}\n\nexport interface ModelPerfProfileStreamRecorder {\n\tmodelKey: (model: Model<Api>) => string | undefined;\n\trecordSample: (modelKey: string, sample: ModelPerfSample) => void;\n\tnowMs?: () => number;\n\tnowIso?: () => string;\n}\n\nexport function isModelPerfProfile(value: unknown): value is ModelPerfProfile {\n\tif (typeof value !== \"object\" || value === null || Array.isArray(value)) return false;\n\tconst record = value as Record<string, unknown>;\n\tconst samples = record.samples;\n\treturn (\n\t\t(record.prefillTokensPerSecond === undefined || isPositiveFiniteNumber(record.prefillTokensPerSecond)) &&\n\t\t(record.decodeTokensPerSecond === undefined || isPositiveFiniteNumber(record.decodeTokensPerSecond)) &&\n\t\t(record.loadMs === undefined || isPositiveFiniteNumber(record.loadMs)) &&\n\t\ttypeof samples === \"number\" &&\n\t\tNumber.isInteger(samples) &&\n\t\tsamples >= 0 &&\n\t\ttypeof record.updatedAt === \"string\"\n\t);\n}\n\nexport function hasUsableModelPerfSample(sample: ModelPerfSample): boolean {\n\treturn (\n\t\tprefillRateFromSample(sample) !== undefined ||\n\t\tdecodeRateFromSample(sample) !== undefined ||\n\t\tisPositiveFiniteNumber(sample.loadMs)\n\t);\n}\n\nexport function updateModelPerfProfile(\n\tcurrent: ModelPerfProfile | undefined,\n\tsample: ModelPerfSample,\n\tat: string = sample.at ?? new Date().toISOString(),\n): ModelPerfProfile | undefined {\n\tconst prefillRate = prefillRateFromSample(sample);\n\tconst decodeRate = decodeRateFromSample(sample);\n\tconst loadMs = isPositiveFiniteNumber(sample.loadMs) ? sample.loadMs : undefined;\n\tif (prefillRate === undefined && decodeRate === undefined && loadMs === undefined) return current;\n\n\tconst previousSamples = current?.samples ?? 0;\n\treturn {\n\t\t...(prefillRate !== undefined && {\n\t\t\tprefillTokensPerSecond: updateEwma(current?.prefillTokensPerSecond, prefillRate, previousSamples),\n\t\t}),\n\t\t...(prefillRate === undefined && current?.prefillTokensPerSecond !== undefined\n\t\t\t? { prefillTokensPerSecond: current.prefillTokensPerSecond }\n\t\t\t: {}),\n\t\t...(decodeRate !== undefined && {\n\t\t\tdecodeTokensPerSecond: updateEwma(current?.decodeTokensPerSecond, decodeRate, previousSamples),\n\t\t}),\n\t\t...(decodeRate === undefined && current?.decodeTokensPerSecond !== undefined\n\t\t\t? { decodeTokensPerSecond: current.decodeTokensPerSecond }\n\t\t\t: {}),\n\t\t...(loadMs !== undefined && { loadMs: updateEwma(current?.loadMs, loadMs, previousSamples) }),\n\t\t...(loadMs === undefined && current?.loadMs !== undefined ? { loadMs: current.loadMs } : {}),\n\t\tsamples: previousSamples + 1,\n\t\tupdatedAt: at,\n\t};\n}\n\nexport function resolveAdaptiveStreamIdleOptions(input: AdaptiveStreamIdleInput): Partial<StreamIdleOptions> {\n\tconst profile = input.profile;\n\tconst result: Partial<StreamIdleOptions> = {};\n\tconst localConnectDefaultMs = input.localClass\n\t\t? Math.max(input.base.connectMs, input.base.quietIdleMs)\n\t\t: input.base.connectMs;\n\tconst quietCeilingMs = Math.max(input.base.quietIdleMs, input.ceilingMs ?? DEFAULT_ADAPTIVE_STREAM_IDLE_CEILING_MS);\n\tconst connectCeilingMs = Math.max(localConnectDefaultMs, input.ceilingMs ?? DEFAULT_ADAPTIVE_STREAM_IDLE_CEILING_MS);\n\n\tconst expectedPrefillMs = expectedPrefillFromProfile(profile, input.promptTokens);\n\tif (expectedPrefillMs !== undefined) {\n\t\tconst quietIdleMs = adaptiveBound(input.base.quietIdleMs, quietCeilingMs, expectedPrefillMs);\n\t\tif (quietIdleMs > input.base.quietIdleMs) result.quietIdleMs = quietIdleMs;\n\t}\n\n\tif (input.localClass) {\n\t\tconst expectedConnectWorkMs =\n\t\t\texpectedPrefillMs !== undefined ? expectedPrefillMs + (profile?.loadMs ?? 0) : profile?.loadMs;\n\t\tconst connectMs =\n\t\t\texpectedConnectWorkMs !== undefined\n\t\t\t\t? adaptiveBound(localConnectDefaultMs, connectCeilingMs, expectedConnectWorkMs)\n\t\t\t\t: localConnectDefaultMs;\n\t\tif (connectMs > input.base.connectMs) result.connectMs = connectMs;\n\t}\n\n\treturn result;\n}\n\nexport function estimateContextPromptTokens(context: Context): number {\n\tconst serialized = JSON.stringify({\n\t\tsystemPrompt: context.systemPrompt,\n\t\tmessages: context.messages,\n\t\ttools: context.tools,\n\t});\n\tif (!serialized) return 0;\n\treturn Math.ceil(serialized.length / CHARS_PER_TOKEN_ESTIMATE);\n}\n\nexport function withModelPerfProfile(streamFn: StreamFn, recorder: ModelPerfProfileStreamRecorder): StreamFn {\n\treturn async (model, context, streamOptions) => {\n\t\tconst nowMs = recorder.nowMs ?? Date.now;\n\t\tconst nowIso = recorder.nowIso ?? (() => new Date().toISOString());\n\t\tconst requestStartedAtMs = nowMs();\n\t\tlet responseHeadersAtMs: number | undefined;\n\t\tlet firstTokenAtMs: number | undefined;\n\t\tconst originalOnResponse = streamOptions?.onResponse;\n\t\tconst outer = createAssistantMessageEventStream();\n\t\tlet latest = emptyAssistantMessage(model);\n\t\tlet inner: Awaited<ReturnType<StreamFn>>;\n\t\ttry {\n\t\t\tinner = await streamFn(model, context, {\n\t\t\t\t...streamOptions,\n\t\t\t\tonResponse: async (response: ProviderResponse, responseModel: Model<Api>) => {\n\t\t\t\t\tresponseHeadersAtMs ??= nowMs();\n\t\t\t\t\tawait originalOnResponse?.(response, responseModel);\n\t\t\t\t},\n\t\t\t});\n\t\t} catch (error) {\n\t\t\touter.push(perfStreamFailure(latest, streamOptions?.signal?.aborted === true, error));\n\t\t\treturn outer;\n\t\t}\n\n\t\tvoid (async () => {\n\t\t\tlet terminalPushed = false;\n\t\t\ttry {\n\t\t\t\tfor await (const event of inner) {\n\t\t\t\t\tlatest = assistantMessageFromEvent(event);\n\t\t\t\t\tif (firstTokenAtMs === undefined && isFirstTokenEvent(event)) {\n\t\t\t\t\t\tfirstTokenAtMs = nowMs();\n\t\t\t\t\t}\n\t\t\t\t\tif (event.type === \"done\") {\n\t\t\t\t\t\tterminalPushed = true;\n\t\t\t\t\t\trecordSuccessfulStreamSample({\n\t\t\t\t\t\t\tcompletionTokens: event.message.usage.output,\n\t\t\t\t\t\t\tdoneAtMs: nowMs(),\n\t\t\t\t\t\t\tfirstTokenAtMs,\n\t\t\t\t\t\t\tmodel,\n\t\t\t\t\t\t\tnowIso,\n\t\t\t\t\t\t\tpromptTokens:\n\t\t\t\t\t\t\t\tevent.message.usage.input + event.message.usage.cacheRead + event.message.usage.cacheWrite,\n\t\t\t\t\t\t\trecorder,\n\t\t\t\t\t\t\trequestStartedAtMs,\n\t\t\t\t\t\t\tresponseHeadersAtMs,\n\t\t\t\t\t\t});\n\t\t\t\t\t} else if (event.type === \"error\") {\n\t\t\t\t\t\tterminalPushed = true;\n\t\t\t\t\t}\n\t\t\t\t\touter.push(event);\n\t\t\t\t\tif (terminalPushed) return;\n\t\t\t\t}\n\t\t\t\touter.push(perfStreamFailure(latest, streamOptions?.signal?.aborted === true));\n\t\t\t} catch (error) {\n\t\t\t\touter.push(perfStreamFailure(latest, streamOptions?.signal?.aborted === true, error));\n\t\t\t}\n\t\t})();\n\n\t\treturn outer;\n\t};\n}\n\nfunction assistantMessageFromEvent(event: AssistantMessageEvent): AssistantMessage {\n\tif (event.type === \"done\") return event.message;\n\tif (event.type === \"error\") return event.error;\n\treturn event.partial;\n}\n\nfunction emptyAssistantMessage(model: Model<Api>): AssistantMessage {\n\treturn {\n\t\trole: \"assistant\",\n\t\tcontent: [],\n\t\tapi: model.api,\n\t\tprovider: model.provider,\n\t\tmodel: model.id,\n\t\tusage: {\n\t\t\tinput: 0,\n\t\t\toutput: 0,\n\t\t\tcacheRead: 0,\n\t\t\tcacheWrite: 0,\n\t\t\ttotalTokens: 0,\n\t\t\tcost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },\n\t\t},\n\t\tstopReason: \"stop\",\n\t\ttimestamp: Date.now(),\n\t};\n}\n\nfunction perfStreamFailure(latest: AssistantMessage, aborted: boolean, error?: unknown): AssistantMessageEvent {\n\tconst stopReason = aborted ? \"aborted\" : \"error\";\n\tconst detail = error instanceof Error ? `: ${error.message}` : error === undefined ? \"\" : `: ${String(error)}`;\n\treturn {\n\t\ttype: \"error\",\n\t\treason: stopReason,\n\t\terror: {\n\t\t\t...latest,\n\t\t\tstopReason,\n\t\t\terrorMessage: aborted\n\t\t\t\t? `stream aborted before terminal event${detail}`\n\t\t\t\t: `stream ended without terminal event${detail}`,\n\t\t},\n\t};\n}\n\nfunction recordSuccessfulStreamSample(input: {\n\tmodel: Model<Api>;\n\trecorder: ModelPerfProfileStreamRecorder;\n\trequestStartedAtMs: number;\n\tresponseHeadersAtMs: number | undefined;\n\tfirstTokenAtMs: number | undefined;\n\tdoneAtMs: number;\n\tpromptTokens: number;\n\tcompletionTokens: number;\n\tnowIso: () => string;\n}): void {\n\tif (input.firstTokenAtMs === undefined) return;\n\tconst modelKey = input.recorder.modelKey(input.model);\n\tif (!modelKey) return;\n\tconst requestToFirstTokenMs = input.firstTokenAtMs - input.requestStartedAtMs;\n\tconst sample: ModelPerfSample = {\n\t\tpromptTokens: input.promptTokens,\n\t\tcompletionTokens: input.completionTokens,\n\t\trequestToFirstTokenMs,\n\t\tfirstTokenToDoneMs: input.doneAtMs - input.firstTokenAtMs,\n\t\tat: input.nowIso(),\n\t};\n\tif (input.responseHeadersAtMs !== undefined) {\n\t\tconst headersToFirstTokenMs = input.firstTokenAtMs - input.responseHeadersAtMs;\n\t\tconst headersWereEffectivelyDeferred =\n\t\t\trequestToFirstTokenMs >= DEFERRED_HEADERS_MIN_REQUEST_MS &&\n\t\t\theadersToFirstTokenMs <= DEFERRED_HEADERS_MAX_GAP_MS;\n\t\tif (!headersWereEffectivelyDeferred) sample.headersToFirstTokenMs = headersToFirstTokenMs;\n\t}\n\tif (!hasUsableModelPerfSample(sample)) return;\n\ttry {\n\t\tinput.recorder.recordSample(modelKey, sample);\n\t} catch {\n\t\t// Perf profiling must never fail the user turn.\n\t}\n}\n\nfunction isFirstTokenEvent(event: AssistantMessageEvent): boolean {\n\treturn (\n\t\t(event.type === \"text_delta\" || event.type === \"thinking_delta\" || event.type === \"toolcall_delta\") &&\n\t\tevent.delta.length > 0\n\t);\n}\n\nfunction expectedPrefillFromProfile(profile: ModelPerfProfile | undefined, promptTokens: number): number | undefined {\n\tconst prefillRate = profile?.prefillTokensPerSecond;\n\tif (!profile || profile.samples < 1 || !prefillRate || promptTokens <= 0) return undefined;\n\tconst expectedPrefillMs = (promptTokens / prefillRate) * 1000;\n\treturn Number.isFinite(expectedPrefillMs) && expectedPrefillMs > 0 ? expectedPrefillMs : undefined;\n}\n\nfunction adaptiveBound(defaultMs: number, ceilingMs: number, expectedMs: number): number {\n\treturn Math.min(Math.ceil(Math.max(defaultMs, expectedMs * 3)), ceilingMs);\n}\n\nfunction prefillRateFromSample(sample: ModelPerfSample): number | undefined {\n\tconst durationMs = isPositiveFiniteNumber(sample.headersToFirstTokenMs)\n\t\t? sample.headersToFirstTokenMs\n\t\t: sample.requestToFirstTokenMs;\n\treturn tokensPerSecond(sample.promptTokens, durationMs);\n}\n\nfunction decodeRateFromSample(sample: ModelPerfSample): number | undefined {\n\treturn tokensPerSecond(sample.completionTokens, sample.firstTokenToDoneMs);\n}\n\nfunction tokensPerSecond(tokens: number | undefined, durationMs: number | undefined): number | undefined {\n\tif (!isPositiveFiniteNumber(tokens) || !isPositiveFiniteNumber(durationMs)) return undefined;\n\treturn tokens / (durationMs / 1000);\n}\n\nfunction updateEwma(previous: number | undefined, next: number, previousSamples: number): number {\n\tif (previous === undefined || previousSamples === 0) return next;\n\treturn previous * (1 - PERF_EWMA_ALPHA) + next * PERF_EWMA_ALPHA;\n}\n\nfunction isPositiveFiniteNumber(value: unknown): value is number {\n\treturn typeof value === \"number\" && Number.isFinite(value) && value > 0;\n}\n"]}

package/dist/core/models/runtime-arbiter.d.ts

@@ -18,18 +18,29 @@ export interface RuntimeLoadRequest {
     nowMs: number;
     minDwellMs?: number;
     pinActiveModel?: string;
+    /** Adapter identity for this request; set by RuntimeResidencyArbiter before planning. */
+    adapterId?: string;
+    /** When pinActiveModel is set, constrain that pin to this adapter when present. */
+    pinActiveAdapterId?: string;
     reservations?: RuntimeReservation[];
     recentEvictions?: RuntimeEvictionRecord[];
+    /** False performs admission/eviction only; the real model request owns cold loading. Default true. */
+    loadModel?: boolean;
 }
 export interface RuntimeReservation {
     model: string;
     bytes: number;
     priority: number;
+    /** Optional runtime identity; omitted means any adapter serving this model. */
+    adapterId?: string;
 }
 export interface RuntimeEvictionRecord {
     evicted: string;
     loaded: string;
     atMs: number;
+    /** Optional identities preserve anti-thrash separation across runtime adapters. */
+    evictedAdapterId?: string;
+    loadedAdapterId?: string;
 }
 export type RuntimeResidencyPlan = {
     status: "fits";

package/dist/core/models/runtime-arbiter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runtime-arbiter.d.ts","sourceRoot":"","sources":["../../../src/core/models/runtime-arbiter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE7E,MAAM,MAAM,gBAAgB,GAAG,MAAM,GAAG,YAAY,GAAG,UAAU,CAAC;AAClE,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,CAAC;AAElE,MAAM,WAAW,oBAAoB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,gBAAgB,EAAE,gBAAgB,CAAC;CACnC;AAED,MAAM,WAAW,kBAAkB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,kBAAkB,EAAE,CAAC;IACpC,eAAe,CAAC,EAAE,qBAAqB,EAAE,CAAC;CAC1C;AAED,MAAM,WAAW,kBAAkB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,qBAAqB;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;CACb;AAED,MAAM,MAAM,oBAAoB,GAC7B;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,oBAAoB,EAAE,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,GAC7F;IAAE,MAAM,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,oBAAoB,EAAE,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,CAAC;AAEnH,MAAM,WAAW,uBAAuB;IACvC,EAAE,EAAE,MAAM,CAAC;IACX,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,IAAI,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAAC;IACxC,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7C,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACtC;AAED,qBAAa,uBAAuB;IACnC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAuC;IAEhE,YAAY,IAAI,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,uBAAuB,EAAE,CAAA;KAAE,EAG7E;IAEK,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAmBlG;CACD;AAED,qBAAa,6BAA8B,YAAW,uBAAuB;IAC5E,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,gBAAgB,eAAyB;IAClD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmF;IAC3G,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAe;IAErC,YACC,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,IAAI,CAAC,aAAa,EAAE,oBAAoB,GAAG,gBAAgB,GAAG,iBAAiB,CAAC,EACzF,OAAO,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,MAAM,CAAA;KAAO,EAKtC;IAEK,IAAI,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAU5C;IAEK,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGjD;IAEK,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE1C;CACD;AAED,qBAAa,mCAAoC,YAAW,uBAAuB;IAClF,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,gBAAgB,SAAmB;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAyD;IACjF,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAe;IAErC,YACC,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,IAAI,CAAC,mBAAmB,EAAE,QAAQ,GAAG,OAAO,GAAG,MAAM,CAAC,EAC/D,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,OAAO,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,MAAM,CAAA;KAAO,EAOtC;IAEK,IAAI,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAa5C;IAEK,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGlD;IAEK,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE3C;CACD;AAED,qBAAa,+BAAgC,YAAW,uBAAuB;IAC9E,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,gBAAgB,aAAuB;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAyB;IAEnD,YAAY,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,oBAAoB,EAAE,WAAW,GAAG,kBAAkB,CAAC,EAAE,EAOhG;IAEK,IAAI,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAE5C;IAEK,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAG;IAEhD,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAG;CAC/C;AAED,wBAAgB,oBAAoB,CAAC,IAAI,EAAE;IAC1C,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,SAAS,oBAAoB,EAAE,CAAC;IAC3C,OAAO,EAAE,kBAAkB,CAAC;CAC5B,GAAG,oBAAoB,CAkDvB","sourcesContent":["import type { OllamaRuntime, TransformersRuntime } from \"./local-runtime.ts\";\n\nexport type ResidencyControl = \"full\" | \"keep-alive\" | \"advisory\";\nexport type RuntimeRole = \"active\" | \"router\" | \"bench\" | \"probe\";\n\nexport interface RuntimeResidentModel {\n\tadapterId: string;\n\tmodel: string;\n\tbytes: number;\n\tlastUsedAtMs: number;\n\tpriority: number;\n\tpinned?: boolean;\n\tresidencyControl: ResidencyControl;\n}\n\nexport interface RuntimeLoadRequest {\n\tmodel: string;\n\tbytes: number;\n\trole: RuntimeRole;\n\tpriority: number;\n\tnowMs: number;\n\tminDwellMs?: number;\n\tpinActiveModel?: string;\n\treservations?: RuntimeReservation[];\n\trecentEvictions?: RuntimeEvictionRecord[];\n}\n\nexport interface RuntimeReservation {\n\tmodel: string;\n\tbytes: number;\n\tpriority: number;\n}\n\nexport interface RuntimeEvictionRecord {\n\tevicted: string;\n\tloaded: string;\n\tatMs: number;\n}\n\nexport type RuntimeResidencyPlan =\n\t| { status: \"fits\"; evict: RuntimeResidentModel[]; requiredBytes: number; budgetBytes: number }\n\t| { status: \"refuse\"; reason: string; evict: RuntimeResidentModel[]; requiredBytes: number; budgetBytes: number };\n\nexport interface RuntimeResidencyAdapter {\n\tid: string;\n\tresidencyControl: ResidencyControl;\n\tlist(): Promise<RuntimeResidentModel[]>;\n\tensureResident(model: string): Promise<void>;\n\trelease(model: string): Promise<void>;\n}\n\nexport class RuntimeResidencyArbiter {\n\tprivate readonly budgetBytes: number;\n\tprivate readonly adapters: Map<string, RuntimeResidencyAdapter>;\n\n\tconstructor(args: { budgetBytes: number; adapters: RuntimeResidencyAdapter[] }) {\n\t\tthis.budgetBytes = args.budgetBytes;\n\t\tthis.adapters = new Map(args.adapters.map((adapter) => [adapter.id, adapter]));\n\t}\n\n\tasync ensureResident(adapterId: string, request: RuntimeLoadRequest): Promise<RuntimeResidencyPlan> {\n\t\tconst adapter = this.adapters.get(adapterId);\n\t\tconst residents = (await Promise.all([...this.adapters.values()].map((entry) => entry.list()))).flat();\n\t\tif (!adapter) {\n\t\t\treturn {\n\t\t\t\tstatus: \"refuse\",\n\t\t\t\treason: \"unknown-adapter\",\n\t\t\t\tevict: [],\n\t\t\t\trequiredBytes: request.bytes,\n\t\t\t\tbudgetBytes: this.budgetBytes,\n\t\t\t};\n\t\t}\n\t\tconst plan = planRuntimeResidency({ budgetBytes: this.budgetBytes, residents, request });\n\t\tif (plan.status !== \"fits\") return plan;\n\t\tfor (const resident of plan.evict) {\n\t\t\tawait this.adapters.get(resident.adapterId)?.release(resident.model);\n\t\t}\n\t\tawait adapter.ensureResident(request.model);\n\t\treturn plan;\n\t}\n}\n\nexport class OllamaRuntimeResidencyAdapter implements RuntimeResidencyAdapter {\n\treadonly id: string;\n\treadonly residencyControl = \"keep-alive\" as const;\n\tprivate readonly runtime: Pick<OllamaRuntime, \"listResidentModels\" | \"ensureResident\" | \"releaseResident\">;\n\tprivate readonly nowMs: () => number;\n\n\tconstructor(\n\t\tid: string,\n\t\truntime: Pick<OllamaRuntime, \"listResidentModels\" | \"ensureResident\" | \"releaseResident\">,\n\t\toptions: { nowMs?: () => number } = {},\n\t) {\n\t\tthis.id = id;\n\t\tthis.runtime = runtime;\n\t\tthis.nowMs = options.nowMs ?? Date.now;\n\t}\n\n\tasync list(): Promise<RuntimeResidentModel[]> {\n\t\tconst now = this.nowMs();\n\t\treturn (await this.runtime.listResidentModels()).map((model) => ({\n\t\t\tadapterId: this.id,\n\t\t\tmodel: model.name,\n\t\t\tbytes: model.sizeBytes,\n\t\t\tlastUsedAtMs: now,\n\t\t\tpriority: 10,\n\t\t\tresidencyControl: this.residencyControl,\n\t\t}));\n\t}\n\n\tasync ensureResident(model: string): Promise<void> {\n\t\tconst result = await this.runtime.ensureResident(model);\n\t\tif (!result.ok) throw new Error(result.error);\n\t}\n\n\tasync release(model: string): Promise<void> {\n\t\tawait this.runtime.releaseResident(model);\n\t}\n}\n\nexport class TransformersRuntimeResidencyAdapter implements RuntimeResidencyAdapter {\n\treadonly id: string;\n\treadonly residencyControl = \"full\" as const;\n\tprivate readonly runtime: Pick<TransformersRuntime, \"detect\" | \"start\" | \"stop\">;\n\tprivate readonly model: string;\n\tprivate readonly bytes: number;\n\tprivate readonly nowMs: () => number;\n\n\tconstructor(\n\t\tid: string,\n\t\truntime: Pick<TransformersRuntime, \"detect\" | \"start\" | \"stop\">,\n\t\tmodel: string,\n\t\tbytes: number,\n\t\toptions: { nowMs?: () => number } = {},\n\t) {\n\t\tthis.id = id;\n\t\tthis.runtime = runtime;\n\t\tthis.model = model;\n\t\tthis.bytes = bytes;\n\t\tthis.nowMs = options.nowMs ?? Date.now;\n\t}\n\n\tasync list(): Promise<RuntimeResidentModel[]> {\n\t\tconst status = await this.runtime.detect();\n\t\tif (!status.serverUp) return [];\n\t\treturn [\n\t\t\t{\n\t\t\t\tadapterId: this.id,\n\t\t\t\tmodel: this.model,\n\t\t\t\tbytes: this.bytes,\n\t\t\t\tlastUsedAtMs: this.nowMs(),\n\t\t\t\tpriority: 10,\n\t\t\t\tresidencyControl: this.residencyControl,\n\t\t\t},\n\t\t];\n\t}\n\n\tasync ensureResident(_model: string): Promise<void> {\n\t\tconst result = await this.runtime.start();\n\t\tif (!result.started && result.reason !== \"already_running\") throw new Error(result.reason);\n\t}\n\n\tasync release(_model: string): Promise<void> {\n\t\tthis.runtime.stop();\n\t}\n}\n\nexport class AdvisoryRuntimeResidencyAdapter implements RuntimeResidencyAdapter {\n\treadonly id: string;\n\treadonly residencyControl = \"advisory\" as const;\n\tprivate readonly residents: RuntimeResidentModel[];\n\n\tconstructor(id: string, residents: Omit<RuntimeResidentModel, \"adapterId\" | \"residencyControl\">[]) {\n\t\tthis.id = id;\n\t\tthis.residents = residents.map((resident) => ({\n\t\t\t...resident,\n\t\t\tadapterId: id,\n\t\t\tresidencyControl: this.residencyControl,\n\t\t}));\n\t}\n\n\tasync list(): Promise<RuntimeResidentModel[]> {\n\t\treturn [...this.residents];\n\t}\n\n\tasync ensureResident(_model: string): Promise<void> {}\n\n\tasync release(_model: string): Promise<void> {}\n}\n\nexport function planRuntimeResidency(args: {\n\tbudgetBytes: number;\n\tresidents: readonly RuntimeResidentModel[];\n\trequest: RuntimeLoadRequest;\n}): RuntimeResidencyPlan {\n\tconst requestedModels = new Map<string, { bytes: number; priority: number }>();\n\trequestedModels.set(args.request.model, { bytes: args.request.bytes, priority: args.request.priority });\n\tfor (const reservation of args.request.reservations ?? []) {\n\t\tconst existing = requestedModels.get(reservation.model);\n\t\tif (!existing || reservation.bytes > existing.bytes) {\n\t\t\trequestedModels.set(reservation.model, { bytes: reservation.bytes, priority: reservation.priority });\n\t\t}\n\t}\n\n\tconst residentByModel = new Map(args.residents.map((resident) => [resident.model, resident]));\n\tconst residentBytes = args.residents.reduce((sum, resident) => sum + resident.bytes, 0);\n\tconst missingBytes = [...requestedModels].reduce((sum, [model, request]) => {\n\t\treturn residentByModel.has(model) ? sum : sum + request.bytes;\n\t}, 0);\n\tconst requiredBytes = residentBytes + missingBytes;\n\tif (requiredBytes <= args.budgetBytes) {\n\t\treturn { status: \"fits\", evict: [], requiredBytes, budgetBytes: args.budgetBytes };\n\t}\n\n\tif (hasPingPongRisk(args.request)) {\n\t\treturn { status: \"refuse\", reason: \"anti-thrash\", evict: [], requiredBytes, budgetBytes: args.budgetBytes };\n\t}\n\n\tlet projectedBytes = requiredBytes;\n\tconst evict: RuntimeResidentModel[] = [];\n\tconst candidates = args.residents\n\t\t.filter((resident) => !requestedModels.has(resident.model))\n\t\t.filter((resident) => !resident.pinned && resident.model !== args.request.pinActiveModel)\n\t\t.filter((resident) => resident.residencyControl !== \"advisory\")\n\t\t.filter(\n\t\t\t(resident) =>\n\t\t\t\targs.request.minDwellMs === undefined ||\n\t\t\t\targs.request.nowMs - resident.lastUsedAtMs >= args.request.minDwellMs,\n\t\t)\n\t\t.sort((left, right) => left.priority - right.priority || left.lastUsedAtMs - right.lastUsedAtMs);\n\tfor (const candidate of candidates) {\n\t\tevict.push(candidate);\n\t\tprojectedBytes -= candidate.bytes;\n\t\tif (projectedBytes <= args.budgetBytes) {\n\t\t\treturn { status: \"fits\", evict, requiredBytes, budgetBytes: args.budgetBytes };\n\t\t}\n\t}\n\treturn {\n\t\tstatus: \"refuse\",\n\t\treason: \"insufficient-evictable-memory\",\n\t\tevict,\n\t\trequiredBytes,\n\t\tbudgetBytes: args.budgetBytes,\n\t};\n}\n\nfunction hasPingPongRisk(request: RuntimeLoadRequest): boolean {\n\treturn (request.recentEvictions ?? []).some(\n\t\t(record) => record.evicted === request.model && request.nowMs - record.atMs <= (request.minDwellMs ?? 0),\n\t);\n}\n"]}
+{"version":3,"file":"runtime-arbiter.d.ts","sourceRoot":"","sources":["../../../src/core/models/runtime-arbiter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE7E,MAAM,MAAM,gBAAgB,GAAG,MAAM,GAAG,YAAY,GAAG,UAAU,CAAC;AAClE,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,CAAC;AAElE,MAAM,WAAW,oBAAoB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,gBAAgB,EAAE,gBAAgB,CAAC;CACnC;AAED,MAAM,WAAW,kBAAkB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,yFAAyF;IACzF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mFAAmF;IACnF,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,YAAY,CAAC,EAAE,kBAAkB,EAAE,CAAC;IACpC,eAAe,CAAC,EAAE,qBAAqB,EAAE,CAAC;IAC1C,sGAAsG;IACtG,SAAS,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,kBAAkB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,+EAA+E;IAC/E,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,qBAAqB;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,mFAAmF;IACnF,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,MAAM,oBAAoB,GAC7B;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,oBAAoB,EAAE,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,GAC7F;IAAE,MAAM,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,oBAAoB,EAAE,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,CAAC;AAEnH,MAAM,WAAW,uBAAuB;IACvC,EAAE,EAAE,MAAM,CAAC;IACX,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,IAAI,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAAC;IACxC,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7C,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACtC;AAED,qBAAa,uBAAuB;IACnC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAuC;IAEhE,YAAY,IAAI,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,uBAAuB,EAAE,CAAA;KAAE,EAG7E;IAEK,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CA6BlG;CACD;AAED,qBAAa,6BAA8B,YAAW,uBAAuB;IAC5E,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,gBAAgB,eAAyB;IAClD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmF;IAC3G,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAe;IAErC,YACC,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,IAAI,CAAC,aAAa,EAAE,oBAAoB,GAAG,gBAAgB,GAAG,iBAAiB,CAAC,EACzF,OAAO,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,MAAM,CAAA;KAAO,EAKtC;IAEK,IAAI,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAU5C;IAEK,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGjD;IAEK,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE1C;CACD;AAED,qBAAa,mCAAoC,YAAW,uBAAuB;IAClF,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,gBAAgB,SAAmB;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAyD;IACjF,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAe;IAErC,YACC,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,IAAI,CAAC,mBAAmB,EAAE,QAAQ,GAAG,OAAO,GAAG,MAAM,CAAC,EAC/D,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,OAAO,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,MAAM,CAAA;KAAO,EAOtC;IAEK,IAAI,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAa5C;IAEK,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGlD;IAEK,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE3C;CACD;AAED,qBAAa,+BAAgC,YAAW,uBAAuB;IAC9E,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,gBAAgB,aAAuB;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAyB;IAEnD,YAAY,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,oBAAoB,EAAE,WAAW,GAAG,kBAAkB,CAAC,EAAE,EAOhG;IAEK,IAAI,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAE5C;IAEK,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAG;IAEhD,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAG;CAC/C;AAED,wBAAgB,oBAAoB,CAAC,IAAI,EAAE;IAC1C,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,SAAS,oBAAoB,EAAE,CAAC;IAC3C,OAAO,EAAE,kBAAkB,CAAC;CAC5B,GAAG,oBAAoB,CAsEvB","sourcesContent":["import type { OllamaRuntime, TransformersRuntime } from \"./local-runtime.ts\";\n\nexport type ResidencyControl = \"full\" | \"keep-alive\" | \"advisory\";\nexport type RuntimeRole = \"active\" | \"router\" | \"bench\" | \"probe\";\n\nexport interface RuntimeResidentModel {\n\tadapterId: string;\n\tmodel: string;\n\tbytes: number;\n\tlastUsedAtMs: number;\n\tpriority: number;\n\tpinned?: boolean;\n\tresidencyControl: ResidencyControl;\n}\n\nexport interface RuntimeLoadRequest {\n\tmodel: string;\n\tbytes: number;\n\trole: RuntimeRole;\n\tpriority: number;\n\tnowMs: number;\n\tminDwellMs?: number;\n\tpinActiveModel?: string;\n\t/** Adapter identity for this request; set by RuntimeResidencyArbiter before planning. */\n\tadapterId?: string;\n\t/** When pinActiveModel is set, constrain that pin to this adapter when present. */\n\tpinActiveAdapterId?: string;\n\treservations?: RuntimeReservation[];\n\trecentEvictions?: RuntimeEvictionRecord[];\n\t/** False performs admission/eviction only; the real model request owns cold loading. Default true. */\n\tloadModel?: boolean;\n}\n\nexport interface RuntimeReservation {\n\tmodel: string;\n\tbytes: number;\n\tpriority: number;\n\t/** Optional runtime identity; omitted means any adapter serving this model. */\n\tadapterId?: string;\n}\n\nexport interface RuntimeEvictionRecord {\n\tevicted: string;\n\tloaded: string;\n\tatMs: number;\n\t/** Optional identities preserve anti-thrash separation across runtime adapters. */\n\tevictedAdapterId?: string;\n\tloadedAdapterId?: string;\n}\n\nexport type RuntimeResidencyPlan =\n\t| { status: \"fits\"; evict: RuntimeResidentModel[]; requiredBytes: number; budgetBytes: number }\n\t| { status: \"refuse\"; reason: string; evict: RuntimeResidentModel[]; requiredBytes: number; budgetBytes: number };\n\nexport interface RuntimeResidencyAdapter {\n\tid: string;\n\tresidencyControl: ResidencyControl;\n\tlist(): Promise<RuntimeResidentModel[]>;\n\tensureResident(model: string): Promise<void>;\n\trelease(model: string): Promise<void>;\n}\n\nexport class RuntimeResidencyArbiter {\n\tprivate readonly budgetBytes: number;\n\tprivate readonly adapters: Map<string, RuntimeResidencyAdapter>;\n\n\tconstructor(args: { budgetBytes: number; adapters: RuntimeResidencyAdapter[] }) {\n\t\tthis.budgetBytes = args.budgetBytes;\n\t\tthis.adapters = new Map(args.adapters.map((adapter) => [adapter.id, adapter]));\n\t}\n\n\tasync ensureResident(adapterId: string, request: RuntimeLoadRequest): Promise<RuntimeResidencyPlan> {\n\t\tconst adapter = this.adapters.get(adapterId);\n\t\tconst residents = (await Promise.all([...this.adapters.values()].map((entry) => entry.list()))).flat();\n\t\tif (!adapter) {\n\t\t\treturn {\n\t\t\t\tstatus: \"refuse\",\n\t\t\t\treason: \"unknown-adapter\",\n\t\t\t\tevict: [],\n\t\t\t\trequiredBytes: request.bytes,\n\t\t\t\tbudgetBytes: this.budgetBytes,\n\t\t\t};\n\t\t}\n\t\tconst requestForAdapter: RuntimeLoadRequest = {\n\t\t\t...request,\n\t\t\tadapterId,\n\t\t\t...(request.pinActiveModel !== undefined && request.pinActiveAdapterId === undefined\n\t\t\t\t? { pinActiveAdapterId: adapterId }\n\t\t\t\t: {}),\n\t\t};\n\t\tconst plan = planRuntimeResidency({ budgetBytes: this.budgetBytes, residents, request: requestForAdapter });\n\t\tif (plan.status !== \"fits\") return plan;\n\t\tfor (const resident of plan.evict) {\n\t\t\tawait this.adapters.get(resident.adapterId)?.release(resident.model);\n\t\t}\n\t\tconst alreadyResident = residents.some(\n\t\t\t(resident) => resident.adapterId === adapterId && resident.model === request.model,\n\t\t);\n\t\tif (request.loadModel !== false && !alreadyResident) await adapter.ensureResident(request.model);\n\t\treturn plan;\n\t}\n}\n\nexport class OllamaRuntimeResidencyAdapter implements RuntimeResidencyAdapter {\n\treadonly id: string;\n\treadonly residencyControl = \"keep-alive\" as const;\n\tprivate readonly runtime: Pick<OllamaRuntime, \"listResidentModels\" | \"ensureResident\" | \"releaseResident\">;\n\tprivate readonly nowMs: () => number;\n\n\tconstructor(\n\t\tid: string,\n\t\truntime: Pick<OllamaRuntime, \"listResidentModels\" | \"ensureResident\" | \"releaseResident\">,\n\t\toptions: { nowMs?: () => number } = {},\n\t) {\n\t\tthis.id = id;\n\t\tthis.runtime = runtime;\n\t\tthis.nowMs = options.nowMs ?? Date.now;\n\t}\n\n\tasync list(): Promise<RuntimeResidentModel[]> {\n\t\tconst now = this.nowMs();\n\t\treturn (await this.runtime.listResidentModels()).map((model) => ({\n\t\t\tadapterId: this.id,\n\t\t\tmodel: model.name,\n\t\t\tbytes: model.sizeBytes,\n\t\t\tlastUsedAtMs: now,\n\t\t\tpriority: 10,\n\t\t\tresidencyControl: this.residencyControl,\n\t\t}));\n\t}\n\n\tasync ensureResident(model: string): Promise<void> {\n\t\tconst result = await this.runtime.ensureResident(model);\n\t\tif (!result.ok) throw new Error(result.error);\n\t}\n\n\tasync release(model: string): Promise<void> {\n\t\tawait this.runtime.releaseResident(model);\n\t}\n}\n\nexport class TransformersRuntimeResidencyAdapter implements RuntimeResidencyAdapter {\n\treadonly id: string;\n\treadonly residencyControl = \"full\" as const;\n\tprivate readonly runtime: Pick<TransformersRuntime, \"detect\" | \"start\" | \"stop\">;\n\tprivate readonly model: string;\n\tprivate readonly bytes: number;\n\tprivate readonly nowMs: () => number;\n\n\tconstructor(\n\t\tid: string,\n\t\truntime: Pick<TransformersRuntime, \"detect\" | \"start\" | \"stop\">,\n\t\tmodel: string,\n\t\tbytes: number,\n\t\toptions: { nowMs?: () => number } = {},\n\t) {\n\t\tthis.id = id;\n\t\tthis.runtime = runtime;\n\t\tthis.model = model;\n\t\tthis.bytes = bytes;\n\t\tthis.nowMs = options.nowMs ?? Date.now;\n\t}\n\n\tasync list(): Promise<RuntimeResidentModel[]> {\n\t\tconst status = await this.runtime.detect();\n\t\tif (!status.serverUp) return [];\n\t\treturn [\n\t\t\t{\n\t\t\t\tadapterId: this.id,\n\t\t\t\tmodel: this.model,\n\t\t\t\tbytes: this.bytes,\n\t\t\t\tlastUsedAtMs: this.nowMs(),\n\t\t\t\tpriority: 10,\n\t\t\t\tresidencyControl: this.residencyControl,\n\t\t\t},\n\t\t];\n\t}\n\n\tasync ensureResident(_model: string): Promise<void> {\n\t\tconst result = await this.runtime.start();\n\t\tif (!result.started && result.reason !== \"already_running\") throw new Error(result.reason);\n\t}\n\n\tasync release(_model: string): Promise<void> {\n\t\tthis.runtime.stop();\n\t}\n}\n\nexport class AdvisoryRuntimeResidencyAdapter implements RuntimeResidencyAdapter {\n\treadonly id: string;\n\treadonly residencyControl = \"advisory\" as const;\n\tprivate readonly residents: RuntimeResidentModel[];\n\n\tconstructor(id: string, residents: Omit<RuntimeResidentModel, \"adapterId\" | \"residencyControl\">[]) {\n\t\tthis.id = id;\n\t\tthis.residents = residents.map((resident) => ({\n\t\t\t...resident,\n\t\t\tadapterId: id,\n\t\t\tresidencyControl: this.residencyControl,\n\t\t}));\n\t}\n\n\tasync list(): Promise<RuntimeResidentModel[]> {\n\t\treturn [...this.residents];\n\t}\n\n\tasync ensureResident(_model: string): Promise<void> {}\n\n\tasync release(_model: string): Promise<void> {}\n}\n\nexport function planRuntimeResidency(args: {\n\tbudgetBytes: number;\n\tresidents: readonly RuntimeResidentModel[];\n\trequest: RuntimeLoadRequest;\n}): RuntimeResidencyPlan {\n\tconst requestedModels = new Map<string, { model: string; adapterId?: string; bytes: number; priority: number }>();\n\tconst addRequestedModel = (entry: { model: string; adapterId?: string; bytes: number; priority: number }): void => {\n\t\tconst key = `${entry.adapterId ?? \"*\"}\\0${entry.model}`;\n\t\tconst existing = requestedModels.get(key);\n\t\tif (!existing || entry.bytes > existing.bytes) requestedModels.set(key, entry);\n\t};\n\taddRequestedModel(args.request);\n\tfor (const reservation of args.request.reservations ?? []) {\n\t\taddRequestedModel({\n\t\t\t...reservation,\n\t\t\tadapterId:\n\t\t\t\treservation.adapterId ?? (reservation.model === args.request.model ? args.request.adapterId : undefined),\n\t\t});\n\t}\n\n\tconst isResidentForRequest = (\n\t\tresident: RuntimeResidentModel,\n\t\trequested: { model: string; adapterId?: string },\n\t): boolean =>\n\t\tresident.model === requested.model &&\n\t\t(requested.adapterId === undefined || resident.adapterId === requested.adapterId);\n\tconst residentBytes = args.residents.reduce((sum, resident) => sum + resident.bytes, 0);\n\tconst missingBytes = [...requestedModels.values()].reduce((sum, requested) => {\n\t\treturn args.residents.some((resident) => isResidentForRequest(resident, requested)) ? sum : sum + requested.bytes;\n\t}, 0);\n\tconst requiredBytes = residentBytes + missingBytes;\n\tif (requiredBytes <= args.budgetBytes) {\n\t\treturn { status: \"fits\", evict: [], requiredBytes, budgetBytes: args.budgetBytes };\n\t}\n\n\tif (hasPingPongRisk(args.request)) {\n\t\treturn { status: \"refuse\", reason: \"anti-thrash\", evict: [], requiredBytes, budgetBytes: args.budgetBytes };\n\t}\n\n\tlet projectedBytes = requiredBytes;\n\tconst evict: RuntimeResidentModel[] = [];\n\tconst candidates = args.residents\n\t\t.filter(\n\t\t\t(resident) => ![...requestedModels.values()].some((requested) => isResidentForRequest(resident, requested)),\n\t\t)\n\t\t.filter(\n\t\t\t(resident) =>\n\t\t\t\t!resident.pinned &&\n\t\t\t\t!(\n\t\t\t\t\tresident.model === args.request.pinActiveModel &&\n\t\t\t\t\t(args.request.pinActiveAdapterId === undefined || resident.adapterId === args.request.pinActiveAdapterId)\n\t\t\t\t),\n\t\t)\n\t\t.filter((resident) => resident.residencyControl !== \"advisory\")\n\t\t.filter(\n\t\t\t(resident) =>\n\t\t\t\targs.request.minDwellMs === undefined ||\n\t\t\t\targs.request.nowMs - resident.lastUsedAtMs >= args.request.minDwellMs,\n\t\t)\n\t\t.sort((left, right) => left.priority - right.priority || left.lastUsedAtMs - right.lastUsedAtMs);\n\tfor (const candidate of candidates) {\n\t\tevict.push(candidate);\n\t\tprojectedBytes -= candidate.bytes;\n\t\tif (projectedBytes <= args.budgetBytes) {\n\t\t\treturn { status: \"fits\", evict, requiredBytes, budgetBytes: args.budgetBytes };\n\t\t}\n\t}\n\treturn {\n\t\tstatus: \"refuse\",\n\t\treason: \"insufficient-evictable-memory\",\n\t\tevict,\n\t\trequiredBytes,\n\t\tbudgetBytes: args.budgetBytes,\n\t};\n}\n\nfunction hasPingPongRisk(request: RuntimeLoadRequest): boolean {\n\treturn (request.recentEvictions ?? []).some(\n\t\t(record) =>\n\t\t\trecord.evicted === request.model &&\n\t\t\t(request.adapterId === undefined ||\n\t\t\t\trecord.evictedAdapterId === undefined ||\n\t\t\t\trecord.evictedAdapterId === request.adapterId) &&\n\t\t\trequest.nowMs - record.atMs <= (request.minDwellMs ?? 0),\n\t);\n}\n"]}

package/dist/core/models/runtime-arbiter.js

@@ -17,13 +17,22 @@ export class RuntimeResidencyArbiter {
                 budgetBytes: this.budgetBytes,
             };
         }
-        const plan = planRuntimeResidency({ budgetBytes: this.budgetBytes, residents, request });
+        const requestForAdapter = {
+            ...request,
+            adapterId,
+            ...(request.pinActiveModel !== undefined && request.pinActiveAdapterId === undefined
+                ? { pinActiveAdapterId: adapterId }
+                : {}),
+        };
+        const plan = planRuntimeResidency({ budgetBytes: this.budgetBytes, residents, request: requestForAdapter });
         if (plan.status !== "fits")
             return plan;
         for (const resident of plan.evict) {
             await this.adapters.get(resident.adapterId)?.release(resident.model);
         }
-        await adapter.ensureResident(request.model);
+        const alreadyResident = residents.some((resident) => resident.adapterId === adapterId && resident.model === request.model);
+        if (request.loadModel !== false && !alreadyResident)
+            await adapter.ensureResident(request.model);
         return plan;
     }
 }
@@ -115,17 +124,24 @@ export class AdvisoryRuntimeResidencyAdapter {
 }
 export function planRuntimeResidency(args) {
     const requestedModels = new Map();
-    requestedModels.set(args.request.model, { bytes: args.request.bytes, priority: args.request.priority });
+    const addRequestedModel = (entry) => {
+        const key = `${entry.adapterId ?? "*"}\0${entry.model}`;
+        const existing = requestedModels.get(key);
+        if (!existing || entry.bytes > existing.bytes)
+            requestedModels.set(key, entry);
+    };
+    addRequestedModel(args.request);
     for (const reservation of args.request.reservations ?? []) {
-        const existing = requestedModels.get(reservation.model);
-        if (!existing || reservation.bytes > existing.bytes) {
-            requestedModels.set(reservation.model, { bytes: reservation.bytes, priority: reservation.priority });
-        }
+        addRequestedModel({
+            ...reservation,
+            adapterId: reservation.adapterId ?? (reservation.model === args.request.model ? args.request.adapterId : undefined),
+        });
     }
-    const residentByModel = new Map(args.residents.map((resident) => [resident.model, resident]));
+    const isResidentForRequest = (resident, requested) => resident.model === requested.model &&
+        (requested.adapterId === undefined || resident.adapterId === requested.adapterId);
     const residentBytes = args.residents.reduce((sum, resident) => sum + resident.bytes, 0);
-    const missingBytes = [...requestedModels].reduce((sum, [model, request]) => {
-        return residentByModel.has(model) ? sum : sum + request.bytes;
+    const missingBytes = [...requestedModels.values()].reduce((sum, requested) => {
+        return args.residents.some((resident) => isResidentForRequest(resident, requested)) ? sum : sum + requested.bytes;
     }, 0);
     const requiredBytes = residentBytes + missingBytes;
     if (requiredBytes <= args.budgetBytes) {
@@ -137,8 +153,10 @@ export function planRuntimeResidency(args) {
     let projectedBytes = requiredBytes;
     const evict = [];
     const candidates = args.residents
-        .filter((resident) => !requestedModels.has(resident.model))
-        .filter((resident) => !resident.pinned && resident.model !== args.request.pinActiveModel)
+        .filter((resident) => ![...requestedModels.values()].some((requested) => isResidentForRequest(resident, requested)))
+        .filter((resident) => !resident.pinned &&
+        !(resident.model === args.request.pinActiveModel &&
+            (args.request.pinActiveAdapterId === undefined || resident.adapterId === args.request.pinActiveAdapterId)))
         .filter((resident) => resident.residencyControl !== "advisory")
         .filter((resident) => args.request.minDwellMs === undefined ||
         args.request.nowMs - resident.lastUsedAtMs >= args.request.minDwellMs)
@@ -159,6 +177,10 @@ export function planRuntimeResidency(args) {
     };
 }
 function hasPingPongRisk(request) {
-    return (request.recentEvictions ?? []).some((record) => record.evicted === request.model && request.nowMs - record.atMs <= (request.minDwellMs ?? 0));
+    return (request.recentEvictions ?? []).some((record) => record.evicted === request.model &&
+        (request.adapterId === undefined ||
+            record.evictedAdapterId === undefined ||
+            record.evictedAdapterId === request.adapterId) &&
+        request.nowMs - record.atMs <= (request.minDwellMs ?? 0));
 }
 //# sourceMappingURL=runtime-arbiter.js.map

package/dist/core/models/runtime-arbiter.js.map

@@ -1 +1 @@
-{"version":3,"file":"runtime-arbiter.js","sourceRoot":"","sources":["../../../src/core/models/runtime-arbiter.ts"],"names":[],"mappings":"AAmDA,MAAM,OAAO,uBAAuB;IAClB,WAAW,CAAS;IACpB,QAAQ,CAAuC;IAEhE,YAAY,IAAkE,EAAE;QAC/E,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;QACpC,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAAA,CAC/E;IAED,KAAK,CAAC,cAAc,CAAC,SAAiB,EAAE,OAA2B,EAAiC;QACnG,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC7C,MAAM,SAAS,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACvG,IAAI,CAAC,OAAO,EAAE,CAAC;YACd,OAAO;gBACN,MAAM,EAAE,QAAQ;gBAChB,MAAM,EAAE,iBAAiB;gBACzB,KAAK,EAAE,EAAE;gBACT,aAAa,EAAE,OAAO,CAAC,KAAK;gBAC5B,WAAW,EAAE,IAAI,CAAC,WAAW;aAC7B,CAAC;QACH,CAAC;QACD,MAAM,IAAI,GAAG,oBAAoB,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;QACzF,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM;YAAE,OAAO,IAAI,CAAC;QACxC,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACnC,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACtE,CAAC;QACD,MAAM,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC5C,OAAO,IAAI,CAAC;IAAA,CACZ;CACD;AAED,MAAM,OAAO,6BAA6B;IAChC,EAAE,CAAS;IACX,gBAAgB,GAAG,YAAqB,CAAC;IACjC,OAAO,CAAmF;IAC1F,KAAK,CAAe;IAErC,YACC,EAAU,EACV,OAAyF,EACzF,OAAO,GAA6B,EAAE,EACrC;QACD,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC;IAAA,CACvC;IAED,KAAK,CAAC,IAAI,GAAoC;QAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QACzB,OAAO,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAChE,SAAS,EAAE,IAAI,CAAC,EAAE;YAClB,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,KAAK,EAAE,KAAK,CAAC,SAAS;YACtB,YAAY,EAAE,GAAG;YACjB,QAAQ,EAAE,EAAE;YACZ,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;SACvC,CAAC,CAAC,CAAC;IAAA,CACJ;IAED,KAAK,CAAC,cAAc,CAAC,KAAa,EAAiB;QAClD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QACxD,IAAI,CAAC,MAAM,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAAA,CAC9C;IAED,KAAK,CAAC,OAAO,CAAC,KAAa,EAAiB;QAC3C,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;IAAA,CAC1C;CACD;AAED,MAAM,OAAO,mCAAmC;IACtC,EAAE,CAAS;IACX,gBAAgB,GAAG,MAAe,CAAC;IAC3B,OAAO,CAAyD;IAChE,KAAK,CAAS;IACd,KAAK,CAAS;IACd,KAAK,CAAe;IAErC,YACC,EAAU,EACV,OAA+D,EAC/D,KAAa,EACb,KAAa,EACb,OAAO,GAA6B,EAAE,EACrC;QACD,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC;IAAA,CACvC;IAED,KAAK,CAAC,IAAI,GAAoC;QAC7C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAC3C,IAAI,CAAC,MAAM,CAAC,QAAQ;YAAE,OAAO,EAAE,CAAC;QAChC,OAAO;YACN;gBACC,SAAS,EAAE,IAAI,CAAC,EAAE;gBAClB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,YAAY,EAAE,IAAI,CAAC,KAAK,EAAE;gBAC1B,QAAQ,EAAE,EAAE;gBACZ,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;aACvC;SACD,CAAC;IAAA,CACF;IAED,KAAK,CAAC,cAAc,CAAC,MAAc,EAAiB;QACnD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAC1C,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,MAAM,KAAK,iBAAiB;YAAE,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAAA,CAC3F;IAED,KAAK,CAAC,OAAO,CAAC,MAAc,EAAiB;QAC5C,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IAAA,CACpB;CACD;AAED,MAAM,OAAO,+BAA+B;IAClC,EAAE,CAAS;IACX,gBAAgB,GAAG,UAAmB,CAAC;IAC/B,SAAS,CAAyB;IAEnD,YAAY,EAAU,EAAE,SAAyE,EAAE;QAClG,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAC7C,GAAG,QAAQ;YACX,SAAS,EAAE,EAAE;YACb,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;SACvC,CAAC,CAAC,CAAC;IAAA,CACJ;IAED,KAAK,CAAC,IAAI,GAAoC;QAC7C,OAAO,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;IAAA,CAC3B;IAED,KAAK,CAAC,cAAc,CAAC,MAAc,EAAiB,EAAC,CAAC;IAEtD,KAAK,CAAC,OAAO,CAAC,MAAc,EAAiB,EAAC,CAAC;CAC/C;AAED,MAAM,UAAU,oBAAoB,CAAC,IAIpC,EAAwB;IACxB,MAAM,eAAe,GAAG,IAAI,GAAG,EAA+C,CAAC;IAC/E,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IACxG,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,EAAE,CAAC;QAC3D,MAAM,QAAQ,GAAG,eAAe,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACxD,IAAI,CAAC,QAAQ,IAAI,WAAW,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;YACrD,eAAe,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,QAAQ,EAAE,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAC;QACtG,CAAC;IACF,CAAC;IAED,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC9F,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IACxF,MAAM,YAAY,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC;QAC3E,OAAO,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC;IAAA,CAC9D,EAAE,CAAC,CAAC,CAAC;IACN,MAAM,aAAa,GAAG,aAAa,GAAG,YAAY,CAAC;IACnD,IAAI,aAAa,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;QACvC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,aAAa,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC;IACpF,CAAC;IAED,IAAI,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACnC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,EAAE,EAAE,aAAa,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC;IAC7G,CAAC;IAED,IAAI,cAAc,GAAG,aAAa,CAAC;IACnC,MAAM,KAAK,GAA2B,EAAE,CAAC;IACzC,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS;SAC/B,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;SAC1D,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,KAAK,KAAK,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC;SACxF,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,gBAAgB,KAAK,UAAU,CAAC;SAC9D,MAAM,CACN,CAAC,QAAQ,EAAE,EAAE,CACZ,IAAI,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS;QACrC,IAAI,CAAC,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC,YAAY,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CACtE;SACA,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC;IAClG,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,cAAc,IAAI,SAAS,CAAC,KAAK,CAAC;QAClC,IAAI,cAAc,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACxC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC;QAChF,CAAC;IACF,CAAC;IACD,OAAO;QACN,MAAM,EAAE,QAAQ;QAChB,MAAM,EAAE,+BAA+B;QACvC,KAAK;QACL,aAAa;QACb,WAAW,EAAE,IAAI,CAAC,WAAW;KAC7B,CAAC;AAAA,CACF;AAED,SAAS,eAAe,CAAC,OAA2B,EAAW;IAC9D,OAAO,CAAC,OAAO,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,IAAI,CAC1C,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,KAAK,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,CAAC,CAAC,CACxG,CAAC;AAAA,CACF","sourcesContent":["import type { OllamaRuntime, TransformersRuntime } from \"./local-runtime.ts\";\n\nexport type ResidencyControl = \"full\" | \"keep-alive\" | \"advisory\";\nexport type RuntimeRole = \"active\" | \"router\" | \"bench\" | \"probe\";\n\nexport interface RuntimeResidentModel {\n\tadapterId: string;\n\tmodel: string;\n\tbytes: number;\n\tlastUsedAtMs: number;\n\tpriority: number;\n\tpinned?: boolean;\n\tresidencyControl: ResidencyControl;\n}\n\nexport interface RuntimeLoadRequest {\n\tmodel: string;\n\tbytes: number;\n\trole: RuntimeRole;\n\tpriority: number;\n\tnowMs: number;\n\tminDwellMs?: number;\n\tpinActiveModel?: string;\n\treservations?: RuntimeReservation[];\n\trecentEvictions?: RuntimeEvictionRecord[];\n}\n\nexport interface RuntimeReservation {\n\tmodel: string;\n\tbytes: number;\n\tpriority: number;\n}\n\nexport interface RuntimeEvictionRecord {\n\tevicted: string;\n\tloaded: string;\n\tatMs: number;\n}\n\nexport type RuntimeResidencyPlan =\n\t| { status: \"fits\"; evict: RuntimeResidentModel[]; requiredBytes: number; budgetBytes: number }\n\t| { status: \"refuse\"; reason: string; evict: RuntimeResidentModel[]; requiredBytes: number; budgetBytes: number };\n\nexport interface RuntimeResidencyAdapter {\n\tid: string;\n\tresidencyControl: ResidencyControl;\n\tlist(): Promise<RuntimeResidentModel[]>;\n\tensureResident(model: string): Promise<void>;\n\trelease(model: string): Promise<void>;\n}\n\nexport class RuntimeResidencyArbiter {\n\tprivate readonly budgetBytes: number;\n\tprivate readonly adapters: Map<string, RuntimeResidencyAdapter>;\n\n\tconstructor(args: { budgetBytes: number; adapters: RuntimeResidencyAdapter[] }) {\n\t\tthis.budgetBytes = args.budgetBytes;\n\t\tthis.adapters = new Map(args.adapters.map((adapter) => [adapter.id, adapter]));\n\t}\n\n\tasync ensureResident(adapterId: string, request: RuntimeLoadRequest): Promise<RuntimeResidencyPlan> {\n\t\tconst adapter = this.adapters.get(adapterId);\n\t\tconst residents = (await Promise.all([...this.adapters.values()].map((entry) => entry.list()))).flat();\n\t\tif (!adapter) {\n\t\t\treturn {\n\t\t\t\tstatus: \"refuse\",\n\t\t\t\treason: \"unknown-adapter\",\n\t\t\t\tevict: [],\n\t\t\t\trequiredBytes: request.bytes,\n\t\t\t\tbudgetBytes: this.budgetBytes,\n\t\t\t};\n\t\t}\n\t\tconst plan = planRuntimeResidency({ budgetBytes: this.budgetBytes, residents, request });\n\t\tif (plan.status !== \"fits\") return plan;\n\t\tfor (const resident of plan.evict) {\n\t\t\tawait this.adapters.get(resident.adapterId)?.release(resident.model);\n\t\t}\n\t\tawait adapter.ensureResident(request.model);\n\t\treturn plan;\n\t}\n}\n\nexport class OllamaRuntimeResidencyAdapter implements RuntimeResidencyAdapter {\n\treadonly id: string;\n\treadonly residencyControl = \"keep-alive\" as const;\n\tprivate readonly runtime: Pick<OllamaRuntime, \"listResidentModels\" | \"ensureResident\" | \"releaseResident\">;\n\tprivate readonly nowMs: () => number;\n\n\tconstructor(\n\t\tid: string,\n\t\truntime: Pick<OllamaRuntime, \"listResidentModels\" | \"ensureResident\" | \"releaseResident\">,\n\t\toptions: { nowMs?: () => number } = {},\n\t) {\n\t\tthis.id = id;\n\t\tthis.runtime = runtime;\n\t\tthis.nowMs = options.nowMs ?? Date.now;\n\t}\n\n\tasync list(): Promise<RuntimeResidentModel[]> {\n\t\tconst now = this.nowMs();\n\t\treturn (await this.runtime.listResidentModels()).map((model) => ({\n\t\t\tadapterId: this.id,\n\t\t\tmodel: model.name,\n\t\t\tbytes: model.sizeBytes,\n\t\t\tlastUsedAtMs: now,\n\t\t\tpriority: 10,\n\t\t\tresidencyControl: this.residencyControl,\n\t\t}));\n\t}\n\n\tasync ensureResident(model: string): Promise<void> {\n\t\tconst result = await this.runtime.ensureResident(model);\n\t\tif (!result.ok) throw new Error(result.error);\n\t}\n\n\tasync release(model: string): Promise<void> {\n\t\tawait this.runtime.releaseResident(model);\n\t}\n}\n\nexport class TransformersRuntimeResidencyAdapter implements RuntimeResidencyAdapter {\n\treadonly id: string;\n\treadonly residencyControl = \"full\" as const;\n\tprivate readonly runtime: Pick<TransformersRuntime, \"detect\" | \"start\" | \"stop\">;\n\tprivate readonly model: string;\n\tprivate readonly bytes: number;\n\tprivate readonly nowMs: () => number;\n\n\tconstructor(\n\t\tid: string,\n\t\truntime: Pick<TransformersRuntime, \"detect\" | \"start\" | \"stop\">,\n\t\tmodel: string,\n\t\tbytes: number,\n\t\toptions: { nowMs?: () => number } = {},\n\t) {\n\t\tthis.id = id;\n\t\tthis.runtime = runtime;\n\t\tthis.model = model;\n\t\tthis.bytes = bytes;\n\t\tthis.nowMs = options.nowMs ?? Date.now;\n\t}\n\n\tasync list(): Promise<RuntimeResidentModel[]> {\n\t\tconst status = await this.runtime.detect();\n\t\tif (!status.serverUp) return [];\n\t\treturn [\n\t\t\t{\n\t\t\t\tadapterId: this.id,\n\t\t\t\tmodel: this.model,\n\t\t\t\tbytes: this.bytes,\n\t\t\t\tlastUsedAtMs: this.nowMs(),\n\t\t\t\tpriority: 10,\n\t\t\t\tresidencyControl: this.residencyControl,\n\t\t\t},\n\t\t];\n\t}\n\n\tasync ensureResident(_model: string): Promise<void> {\n\t\tconst result = await this.runtime.start();\n\t\tif (!result.started && result.reason !== \"already_running\") throw new Error(result.reason);\n\t}\n\n\tasync release(_model: string): Promise<void> {\n\t\tthis.runtime.stop();\n\t}\n}\n\nexport class AdvisoryRuntimeResidencyAdapter implements RuntimeResidencyAdapter {\n\treadonly id: string;\n\treadonly residencyControl = \"advisory\" as const;\n\tprivate readonly residents: RuntimeResidentModel[];\n\n\tconstructor(id: string, residents: Omit<RuntimeResidentModel, \"adapterId\" | \"residencyControl\">[]) {\n\t\tthis.id = id;\n\t\tthis.residents = residents.map((resident) => ({\n\t\t\t...resident,\n\t\t\tadapterId: id,\n\t\t\tresidencyControl: this.residencyControl,\n\t\t}));\n\t}\n\n\tasync list(): Promise<RuntimeResidentModel[]> {\n\t\treturn [...this.residents];\n\t}\n\n\tasync ensureResident(_model: string): Promise<void> {}\n\n\tasync release(_model: string): Promise<void> {}\n}\n\nexport function planRuntimeResidency(args: {\n\tbudgetBytes: number;\n\tresidents: readonly RuntimeResidentModel[];\n\trequest: RuntimeLoadRequest;\n}): RuntimeResidencyPlan {\n\tconst requestedModels = new Map<string, { bytes: number; priority: number }>();\n\trequestedModels.set(args.request.model, { bytes: args.request.bytes, priority: args.request.priority });\n\tfor (const reservation of args.request.reservations ?? []) {\n\t\tconst existing = requestedModels.get(reservation.model);\n\t\tif (!existing || reservation.bytes > existing.bytes) {\n\t\t\trequestedModels.set(reservation.model, { bytes: reservation.bytes, priority: reservation.priority });\n\t\t}\n\t}\n\n\tconst residentByModel = new Map(args.residents.map((resident) => [resident.model, resident]));\n\tconst residentBytes = args.residents.reduce((sum, resident) => sum + resident.bytes, 0);\n\tconst missingBytes = [...requestedModels].reduce((sum, [model, request]) => {\n\t\treturn residentByModel.has(model) ? sum : sum + request.bytes;\n\t}, 0);\n\tconst requiredBytes = residentBytes + missingBytes;\n\tif (requiredBytes <= args.budgetBytes) {\n\t\treturn { status: \"fits\", evict: [], requiredBytes, budgetBytes: args.budgetBytes };\n\t}\n\n\tif (hasPingPongRisk(args.request)) {\n\t\treturn { status: \"refuse\", reason: \"anti-thrash\", evict: [], requiredBytes, budgetBytes: args.budgetBytes };\n\t}\n\n\tlet projectedBytes = requiredBytes;\n\tconst evict: RuntimeResidentModel[] = [];\n\tconst candidates = args.residents\n\t\t.filter((resident) => !requestedModels.has(resident.model))\n\t\t.filter((resident) => !resident.pinned && resident.model !== args.request.pinActiveModel)\n\t\t.filter((resident) => resident.residencyControl !== \"advisory\")\n\t\t.filter(\n\t\t\t(resident) =>\n\t\t\t\targs.request.minDwellMs === undefined ||\n\t\t\t\targs.request.nowMs - resident.lastUsedAtMs >= args.request.minDwellMs,\n\t\t)\n\t\t.sort((left, right) => left.priority - right.priority || left.lastUsedAtMs - right.lastUsedAtMs);\n\tfor (const candidate of candidates) {\n\t\tevict.push(candidate);\n\t\tprojectedBytes -= candidate.bytes;\n\t\tif (projectedBytes <= args.budgetBytes) {\n\t\t\treturn { status: \"fits\", evict, requiredBytes, budgetBytes: args.budgetBytes };\n\t\t}\n\t}\n\treturn {\n\t\tstatus: \"refuse\",\n\t\treason: \"insufficient-evictable-memory\",\n\t\tevict,\n\t\trequiredBytes,\n\t\tbudgetBytes: args.budgetBytes,\n\t};\n}\n\nfunction hasPingPongRisk(request: RuntimeLoadRequest): boolean {\n\treturn (request.recentEvictions ?? []).some(\n\t\t(record) => record.evicted === request.model && request.nowMs - record.atMs <= (request.minDwellMs ?? 0),\n\t);\n}\n"]}
+{"version":3,"file":"runtime-arbiter.js","sourceRoot":"","sources":["../../../src/core/models/runtime-arbiter.ts"],"names":[],"mappings":"AA8DA,MAAM,OAAO,uBAAuB;IAClB,WAAW,CAAS;IACpB,QAAQ,CAAuC;IAEhE,YAAY,IAAkE,EAAE;QAC/E,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;QACpC,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAAA,CAC/E;IAED,KAAK,CAAC,cAAc,CAAC,SAAiB,EAAE,OAA2B,EAAiC;QACnG,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC7C,MAAM,SAAS,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACvG,IAAI,CAAC,OAAO,EAAE,CAAC;YACd,OAAO;gBACN,MAAM,EAAE,QAAQ;gBAChB,MAAM,EAAE,iBAAiB;gBACzB,KAAK,EAAE,EAAE;gBACT,aAAa,EAAE,OAAO,CAAC,KAAK;gBAC5B,WAAW,EAAE,IAAI,CAAC,WAAW;aAC7B,CAAC;QACH,CAAC;QACD,MAAM,iBAAiB,GAAuB;YAC7C,GAAG,OAAO;YACV,SAAS;YACT,GAAG,CAAC,OAAO,CAAC,cAAc,KAAK,SAAS,IAAI,OAAO,CAAC,kBAAkB,KAAK,SAAS;gBACnF,CAAC,CAAC,EAAE,kBAAkB,EAAE,SAAS,EAAE;gBACnC,CAAC,CAAC,EAAE,CAAC;SACN,CAAC;QACF,MAAM,IAAI,GAAG,oBAAoB,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,SAAS,EAAE,OAAO,EAAE,iBAAiB,EAAE,CAAC,CAAC;QAC5G,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM;YAAE,OAAO,IAAI,CAAC;QACxC,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACnC,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACtE,CAAC;QACD,MAAM,eAAe,GAAG,SAAS,CAAC,IAAI,CACrC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,SAAS,KAAK,SAAS,IAAI,QAAQ,CAAC,KAAK,KAAK,OAAO,CAAC,KAAK,CAClF,CAAC;QACF,IAAI,OAAO,CAAC,SAAS,KAAK,KAAK,IAAI,CAAC,eAAe;YAAE,MAAM,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACjG,OAAO,IAAI,CAAC;IAAA,CACZ;CACD;AAED,MAAM,OAAO,6BAA6B;IAChC,EAAE,CAAS;IACX,gBAAgB,GAAG,YAAqB,CAAC;IACjC,OAAO,CAAmF;IAC1F,KAAK,CAAe;IAErC,YACC,EAAU,EACV,OAAyF,EACzF,OAAO,GAA6B,EAAE,EACrC;QACD,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC;IAAA,CACvC;IAED,KAAK,CAAC,IAAI,GAAoC;QAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QACzB,OAAO,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAChE,SAAS,EAAE,IAAI,CAAC,EAAE;YAClB,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,KAAK,EAAE,KAAK,CAAC,SAAS;YACtB,YAAY,EAAE,GAAG;YACjB,QAAQ,EAAE,EAAE;YACZ,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;SACvC,CAAC,CAAC,CAAC;IAAA,CACJ;IAED,KAAK,CAAC,cAAc,CAAC,KAAa,EAAiB;QAClD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QACxD,IAAI,CAAC,MAAM,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAAA,CAC9C;IAED,KAAK,CAAC,OAAO,CAAC,KAAa,EAAiB;QAC3C,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;IAAA,CAC1C;CACD;AAED,MAAM,OAAO,mCAAmC;IACtC,EAAE,CAAS;IACX,gBAAgB,GAAG,MAAe,CAAC;IAC3B,OAAO,CAAyD;IAChE,KAAK,CAAS;IACd,KAAK,CAAS;IACd,KAAK,CAAe;IAErC,YACC,EAAU,EACV,OAA+D,EAC/D,KAAa,EACb,KAAa,EACb,OAAO,GAA6B,EAAE,EACrC;QACD,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC;IAAA,CACvC;IAED,KAAK,CAAC,IAAI,GAAoC;QAC7C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAC3C,IAAI,CAAC,MAAM,CAAC,QAAQ;YAAE,OAAO,EAAE,CAAC;QAChC,OAAO;YACN;gBACC,SAAS,EAAE,IAAI,CAAC,EAAE;gBAClB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,YAAY,EAAE,IAAI,CAAC,KAAK,EAAE;gBAC1B,QAAQ,EAAE,EAAE;gBACZ,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;aACvC;SACD,CAAC;IAAA,CACF;IAED,KAAK,CAAC,cAAc,CAAC,MAAc,EAAiB;QACnD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAC1C,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,MAAM,KAAK,iBAAiB;YAAE,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAAA,CAC3F;IAED,KAAK,CAAC,OAAO,CAAC,MAAc,EAAiB;QAC5C,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IAAA,CACpB;CACD;AAED,MAAM,OAAO,+BAA+B;IAClC,EAAE,CAAS;IACX,gBAAgB,GAAG,UAAmB,CAAC;IAC/B,SAAS,CAAyB;IAEnD,YAAY,EAAU,EAAE,SAAyE,EAAE;QAClG,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAC7C,GAAG,QAAQ;YACX,SAAS,EAAE,EAAE;YACb,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;SACvC,CAAC,CAAC,CAAC;IAAA,CACJ;IAED,KAAK,CAAC,IAAI,GAAoC;QAC7C,OAAO,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;IAAA,CAC3B;IAED,KAAK,CAAC,cAAc,CAAC,MAAc,EAAiB,EAAC,CAAC;IAEtD,KAAK,CAAC,OAAO,CAAC,MAAc,EAAiB,EAAC,CAAC;CAC/C;AAED,MAAM,UAAU,oBAAoB,CAAC,IAIpC,EAAwB;IACxB,MAAM,eAAe,GAAG,IAAI,GAAG,EAAkF,CAAC;IAClH,MAAM,iBAAiB,GAAG,CAAC,KAA6E,EAAQ,EAAE,CAAC;QAClH,MAAM,GAAG,GAAG,GAAG,KAAK,CAAC,SAAS,IAAI,GAAG,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC;QACxD,MAAM,QAAQ,GAAG,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC1C,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK;YAAE,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAAA,CAC/E,CAAC;IACF,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAChC,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,EAAE,CAAC;QAC3D,iBAAiB,CAAC;YACjB,GAAG,WAAW;YACd,SAAS,EACR,WAAW,CAAC,SAAS,IAAI,CAAC,WAAW,CAAC,KAAK,KAAK,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;SACzG,CAAC,CAAC;IACJ,CAAC;IAED,MAAM,oBAAoB,GAAG,CAC5B,QAA8B,EAC9B,SAAgD,EACtC,EAAE,CACZ,QAAQ,CAAC,KAAK,KAAK,SAAS,CAAC,KAAK;QAClC,CAAC,SAAS,CAAC,SAAS,KAAK,SAAS,IAAI,QAAQ,CAAC,SAAS,KAAK,SAAS,CAAC,SAAS,CAAC,CAAC;IACnF,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IACxF,MAAM,YAAY,GAAG,CAAC,GAAG,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE,CAAC;QAC7E,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,oBAAoB,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,SAAS,CAAC,KAAK,CAAC;IAAA,CAClH,EAAE,CAAC,CAAC,CAAC;IACN,MAAM,aAAa,GAAG,aAAa,GAAG,YAAY,CAAC;IACnD,IAAI,aAAa,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;QACvC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,aAAa,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC;IACpF,CAAC;IAED,IAAI,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACnC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,EAAE,EAAE,aAAa,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC;IAC7G,CAAC;IAED,IAAI,cAAc,GAAG,aAAa,CAAC;IACnC,MAAM,KAAK,GAA2B,EAAE,CAAC;IACzC,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS;SAC/B,MAAM,CACN,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,oBAAoB,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAC3G;SACA,MAAM,CACN,CAAC,QAAQ,EAAE,EAAE,CACZ,CAAC,QAAQ,CAAC,MAAM;QAChB,CAAC,CACA,QAAQ,CAAC,KAAK,KAAK,IAAI,CAAC,OAAO,CAAC,cAAc;YAC9C,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KAAK,SAAS,IAAI,QAAQ,CAAC,SAAS,KAAK,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CACzG,CACF;SACA,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,gBAAgB,KAAK,UAAU,CAAC;SAC9D,MAAM,CACN,CAAC,QAAQ,EAAE,EAAE,CACZ,IAAI,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS;QACrC,IAAI,CAAC,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC,YAAY,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CACtE;SACA,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC;IAClG,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,cAAc,IAAI,SAAS,CAAC,KAAK,CAAC;QAClC,IAAI,cAAc,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACxC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC;QAChF,CAAC;IACF,CAAC;IACD,OAAO;QACN,MAAM,EAAE,QAAQ;QAChB,MAAM,EAAE,+BAA+B;QACvC,KAAK;QACL,aAAa;QACb,WAAW,EAAE,IAAI,CAAC,WAAW;KAC7B,CAAC;AAAA,CACF;AAED,SAAS,eAAe,CAAC,OAA2B,EAAW;IAC9D,OAAO,CAAC,OAAO,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,IAAI,CAC1C,CAAC,MAAM,EAAE,EAAE,CACV,MAAM,CAAC,OAAO,KAAK,OAAO,CAAC,KAAK;QAChC,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS;YAC/B,MAAM,CAAC,gBAAgB,KAAK,SAAS;YACrC,MAAM,CAAC,gBAAgB,KAAK,OAAO,CAAC,SAAS,CAAC;QAC/C,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,CAAC,CAAC,CACzD,CAAC;AAAA,CACF","sourcesContent":["import type { OllamaRuntime, TransformersRuntime } from \"./local-runtime.ts\";\n\nexport type ResidencyControl = \"full\" | \"keep-alive\" | \"advisory\";\nexport type RuntimeRole = \"active\" | \"router\" | \"bench\" | \"probe\";\n\nexport interface RuntimeResidentModel {\n\tadapterId: string;\n\tmodel: string;\n\tbytes: number;\n\tlastUsedAtMs: number;\n\tpriority: number;\n\tpinned?: boolean;\n\tresidencyControl: ResidencyControl;\n}\n\nexport interface RuntimeLoadRequest {\n\tmodel: string;\n\tbytes: number;\n\trole: RuntimeRole;\n\tpriority: number;\n\tnowMs: number;\n\tminDwellMs?: number;\n\tpinActiveModel?: string;\n\t/** Adapter identity for this request; set by RuntimeResidencyArbiter before planning. */\n\tadapterId?: string;\n\t/** When pinActiveModel is set, constrain that pin to this adapter when present. */\n\tpinActiveAdapterId?: string;\n\treservations?: RuntimeReservation[];\n\trecentEvictions?: RuntimeEvictionRecord[];\n\t/** False performs admission/eviction only; the real model request owns cold loading. Default true. */\n\tloadModel?: boolean;\n}\n\nexport interface RuntimeReservation {\n\tmodel: string;\n\tbytes: number;\n\tpriority: number;\n\t/** Optional runtime identity; omitted means any adapter serving this model. */\n\tadapterId?: string;\n}\n\nexport interface RuntimeEvictionRecord {\n\tevicted: string;\n\tloaded: string;\n\tatMs: number;\n\t/** Optional identities preserve anti-thrash separation across runtime adapters. */\n\tevictedAdapterId?: string;\n\tloadedAdapterId?: string;\n}\n\nexport type RuntimeResidencyPlan =\n\t| { status: \"fits\"; evict: RuntimeResidentModel[]; requiredBytes: number; budgetBytes: number }\n\t| { status: \"refuse\"; reason: string; evict: RuntimeResidentModel[]; requiredBytes: number; budgetBytes: number };\n\nexport interface RuntimeResidencyAdapter {\n\tid: string;\n\tresidencyControl: ResidencyControl;\n\tlist(): Promise<RuntimeResidentModel[]>;\n\tensureResident(model: string): Promise<void>;\n\trelease(model: string): Promise<void>;\n}\n\nexport class RuntimeResidencyArbiter {\n\tprivate readonly budgetBytes: number;\n\tprivate readonly adapters: Map<string, RuntimeResidencyAdapter>;\n\n\tconstructor(args: { budgetBytes: number; adapters: RuntimeResidencyAdapter[] }) {\n\t\tthis.budgetBytes = args.budgetBytes;\n\t\tthis.adapters = new Map(args.adapters.map((adapter) => [adapter.id, adapter]));\n\t}\n\n\tasync ensureResident(adapterId: string, request: RuntimeLoadRequest): Promise<RuntimeResidencyPlan> {\n\t\tconst adapter = this.adapters.get(adapterId);\n\t\tconst residents = (await Promise.all([...this.adapters.values()].map((entry) => entry.list()))).flat();\n\t\tif (!adapter) {\n\t\t\treturn {\n\t\t\t\tstatus: \"refuse\",\n\t\t\t\treason: \"unknown-adapter\",\n\t\t\t\tevict: [],\n\t\t\t\trequiredBytes: request.bytes,\n\t\t\t\tbudgetBytes: this.budgetBytes,\n\t\t\t};\n\t\t}\n\t\tconst requestForAdapter: RuntimeLoadRequest = {\n\t\t\t...request,\n\t\t\tadapterId,\n\t\t\t...(request.pinActiveModel !== undefined && request.pinActiveAdapterId === undefined\n\t\t\t\t? { pinActiveAdapterId: adapterId }\n\t\t\t\t: {}),\n\t\t};\n\t\tconst plan = planRuntimeResidency({ budgetBytes: this.budgetBytes, residents, request: requestForAdapter });\n\t\tif (plan.status !== \"fits\") return plan;\n\t\tfor (const resident of plan.evict) {\n\t\t\tawait this.adapters.get(resident.adapterId)?.release(resident.model);\n\t\t}\n\t\tconst alreadyResident = residents.some(\n\t\t\t(resident) => resident.adapterId === adapterId && resident.model === request.model,\n\t\t);\n\t\tif (request.loadModel !== false && !alreadyResident) await adapter.ensureResident(request.model);\n\t\treturn plan;\n\t}\n}\n\nexport class OllamaRuntimeResidencyAdapter implements RuntimeResidencyAdapter {\n\treadonly id: string;\n\treadonly residencyControl = \"keep-alive\" as const;\n\tprivate readonly runtime: Pick<OllamaRuntime, \"listResidentModels\" | \"ensureResident\" | \"releaseResident\">;\n\tprivate readonly nowMs: () => number;\n\n\tconstructor(\n\t\tid: string,\n\t\truntime: Pick<OllamaRuntime, \"listResidentModels\" | \"ensureResident\" | \"releaseResident\">,\n\t\toptions: { nowMs?: () => number } = {},\n\t) {\n\t\tthis.id = id;\n\t\tthis.runtime = runtime;\n\t\tthis.nowMs = options.nowMs ?? Date.now;\n\t}\n\n\tasync list(): Promise<RuntimeResidentModel[]> {\n\t\tconst now = this.nowMs();\n\t\treturn (await this.runtime.listResidentModels()).map((model) => ({\n\t\t\tadapterId: this.id,\n\t\t\tmodel: model.name,\n\t\t\tbytes: model.sizeBytes,\n\t\t\tlastUsedAtMs: now,\n\t\t\tpriority: 10,\n\t\t\tresidencyControl: this.residencyControl,\n\t\t}));\n\t}\n\n\tasync ensureResident(model: string): Promise<void> {\n\t\tconst result = await this.runtime.ensureResident(model);\n\t\tif (!result.ok) throw new Error(result.error);\n\t}\n\n\tasync release(model: string): Promise<void> {\n\t\tawait this.runtime.releaseResident(model);\n\t}\n}\n\nexport class TransformersRuntimeResidencyAdapter implements RuntimeResidencyAdapter {\n\treadonly id: string;\n\treadonly residencyControl = \"full\" as const;\n\tprivate readonly runtime: Pick<TransformersRuntime, \"detect\" | \"start\" | \"stop\">;\n\tprivate readonly model: string;\n\tprivate readonly bytes: number;\n\tprivate readonly nowMs: () => number;\n\n\tconstructor(\n\t\tid: string,\n\t\truntime: Pick<TransformersRuntime, \"detect\" | \"start\" | \"stop\">,\n\t\tmodel: string,\n\t\tbytes: number,\n\t\toptions: { nowMs?: () => number } = {},\n\t) {\n\t\tthis.id = id;\n\t\tthis.runtime = runtime;\n\t\tthis.model = model;\n\t\tthis.bytes = bytes;\n\t\tthis.nowMs = options.nowMs ?? Date.now;\n\t}\n\n\tasync list(): Promise<RuntimeResidentModel[]> {\n\t\tconst status = await this.runtime.detect();\n\t\tif (!status.serverUp) return [];\n\t\treturn [\n\t\t\t{\n\t\t\t\tadapterId: this.id,\n\t\t\t\tmodel: this.model,\n\t\t\t\tbytes: this.bytes,\n\t\t\t\tlastUsedAtMs: this.nowMs(),\n\t\t\t\tpriority: 10,\n\t\t\t\tresidencyControl: this.residencyControl,\n\t\t\t},\n\t\t];\n\t}\n\n\tasync ensureResident(_model: string): Promise<void> {\n\t\tconst result = await this.runtime.start();\n\t\tif (!result.started && result.reason !== \"already_running\") throw new Error(result.reason);\n\t}\n\n\tasync release(_model: string): Promise<void> {\n\t\tthis.runtime.stop();\n\t}\n}\n\nexport class AdvisoryRuntimeResidencyAdapter implements RuntimeResidencyAdapter {\n\treadonly id: string;\n\treadonly residencyControl = \"advisory\" as const;\n\tprivate readonly residents: RuntimeResidentModel[];\n\n\tconstructor(id: string, residents: Omit<RuntimeResidentModel, \"adapterId\" | \"residencyControl\">[]) {\n\t\tthis.id = id;\n\t\tthis.residents = residents.map((resident) => ({\n\t\t\t...resident,\n\t\t\tadapterId: id,\n\t\t\tresidencyControl: this.residencyControl,\n\t\t}));\n\t}\n\n\tasync list(): Promise<RuntimeResidentModel[]> {\n\t\treturn [...this.residents];\n\t}\n\n\tasync ensureResident(_model: string): Promise<void> {}\n\n\tasync release(_model: string): Promise<void> {}\n}\n\nexport function planRuntimeResidency(args: {\n\tbudgetBytes: number;\n\tresidents: readonly RuntimeResidentModel[];\n\trequest: RuntimeLoadRequest;\n}): RuntimeResidencyPlan {\n\tconst requestedModels = new Map<string, { model: string; adapterId?: string; bytes: number; priority: number }>();\n\tconst addRequestedModel = (entry: { model: string; adapterId?: string; bytes: number; priority: number }): void => {\n\t\tconst key = `${entry.adapterId ?? \"*\"}\\0${entry.model}`;\n\t\tconst existing = requestedModels.get(key);\n\t\tif (!existing || entry.bytes > existing.bytes) requestedModels.set(key, entry);\n\t};\n\taddRequestedModel(args.request);\n\tfor (const reservation of args.request.reservations ?? []) {\n\t\taddRequestedModel({\n\t\t\t...reservation,\n\t\t\tadapterId:\n\t\t\t\treservation.adapterId ?? (reservation.model === args.request.model ? args.request.adapterId : undefined),\n\t\t});\n\t}\n\n\tconst isResidentForRequest = (\n\t\tresident: RuntimeResidentModel,\n\t\trequested: { model: string; adapterId?: string },\n\t): boolean =>\n\t\tresident.model === requested.model &&\n\t\t(requested.adapterId === undefined || resident.adapterId === requested.adapterId);\n\tconst residentBytes = args.residents.reduce((sum, resident) => sum + resident.bytes, 0);\n\tconst missingBytes = [...requestedModels.values()].reduce((sum, requested) => {\n\t\treturn args.residents.some((resident) => isResidentForRequest(resident, requested)) ? sum : sum + requested.bytes;\n\t}, 0);\n\tconst requiredBytes = residentBytes + missingBytes;\n\tif (requiredBytes <= args.budgetBytes) {\n\t\treturn { status: \"fits\", evict: [], requiredBytes, budgetBytes: args.budgetBytes };\n\t}\n\n\tif (hasPingPongRisk(args.request)) {\n\t\treturn { status: \"refuse\", reason: \"anti-thrash\", evict: [], requiredBytes, budgetBytes: args.budgetBytes };\n\t}\n\n\tlet projectedBytes = requiredBytes;\n\tconst evict: RuntimeResidentModel[] = [];\n\tconst candidates = args.residents\n\t\t.filter(\n\t\t\t(resident) => ![...requestedModels.values()].some((requested) => isResidentForRequest(resident, requested)),\n\t\t)\n\t\t.filter(\n\t\t\t(resident) =>\n\t\t\t\t!resident.pinned &&\n\t\t\t\t!(\n\t\t\t\t\tresident.model === args.request.pinActiveModel &&\n\t\t\t\t\t(args.request.pinActiveAdapterId === undefined || resident.adapterId === args.request.pinActiveAdapterId)\n\t\t\t\t),\n\t\t)\n\t\t.filter((resident) => resident.residencyControl !== \"advisory\")\n\t\t.filter(\n\t\t\t(resident) =>\n\t\t\t\targs.request.minDwellMs === undefined ||\n\t\t\t\targs.request.nowMs - resident.lastUsedAtMs >= args.request.minDwellMs,\n\t\t)\n\t\t.sort((left, right) => left.priority - right.priority || left.lastUsedAtMs - right.lastUsedAtMs);\n\tfor (const candidate of candidates) {\n\t\tevict.push(candidate);\n\t\tprojectedBytes -= candidate.bytes;\n\t\tif (projectedBytes <= args.budgetBytes) {\n\t\t\treturn { status: \"fits\", evict, requiredBytes, budgetBytes: args.budgetBytes };\n\t\t}\n\t}\n\treturn {\n\t\tstatus: \"refuse\",\n\t\treason: \"insufficient-evictable-memory\",\n\t\tevict,\n\t\trequiredBytes,\n\t\tbudgetBytes: args.budgetBytes,\n\t};\n}\n\nfunction hasPingPongRisk(request: RuntimeLoadRequest): boolean {\n\treturn (request.recentEvictions ?? []).some(\n\t\t(record) =>\n\t\t\trecord.evicted === request.model &&\n\t\t\t(request.adapterId === undefined ||\n\t\t\t\trecord.evictedAdapterId === undefined ||\n\t\t\t\trecord.evictedAdapterId === request.adapterId) &&\n\t\t\trequest.nowMs - record.atMs <= (request.minDwellMs ?? 0),\n\t);\n}\n"]}

package/dist/core/profile-filter-controller.d.ts

@@ -33,15 +33,26 @@ export interface ProfileFilterControllerDeps {
     /** Apply a thinking level (delegates to model-selection). */
     setThinkingLevel(level: ThinkingLevel): void;
 }
+export interface ProfileFilterReloadSnapshot {
+    inertExtensionWarnings: string[];
+    profileDeniedExtensionCount: number;
+    profileControlledModel: boolean;
+    profileControlledThinking: boolean;
+}
 export declare class ProfileFilterController {
     /** G12: extensions loaded but rendered fully inert by the profile's tools filter. */
     private _inertExtensionWarnings;
     /** Count of extensions withheld by the active resource profile (for the /context observation). */
     private _profileDeniedExtensionCount;
+    /** Whether the last applied profile generation, rather than the base session selection, owns the field. */
+    private _profileControlledModel;
+    private _profileControlledThinking;
     private readonly deps;
     constructor(deps: ProfileFilterControllerDeps);
     /** Inert-extension warnings tracked by the last {@link filterExtensionsForRuntime} pass. */
     getInertExtensionWarnings(): string[];
+    createReloadSnapshot(): ProfileFilterReloadSnapshot;
+    restoreReloadSnapshot(snapshot: ProfileFilterReloadSnapshot): void;
     /**
      * Resolve the active resource-profile tool allow/block filter from current settings.
      * Mirrors the construction-time derivation (settingsManager.getResourceProfileFilter("tools"))