@caupulican/pi-adaptative 0.81.20 → 0.81.22

package/dist/core/models/perf-profile.js.map

@@ -1 +1 @@
-{"version":3,"file":"perf-profile.js","sourceRoot":"","sources":["../../../src/core/models/perf-profile.ts"],"names":[],"mappings":"AACA,OAAO,EAIN,iCAAiC,GAGjC,MAAM,mBAAmB,CAAC;AAE3B,MAAM,eAAe,GAAG,GAAG,CAAC;AAC5B,MAAM,wBAAwB,GAAG,CAAC,CAAC;AACnC,MAAM,CAAC,MAAM,uCAAuC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAgCtE,MAAM,UAAU,kBAAkB,CAAC,KAAc,EAA6B;IAC7E,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACtF,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;IAC/B,OAAO,CACN,CAAC,MAAM,CAAC,sBAAsB,KAAK,SAAS,IAAI,sBAAsB,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;QACtG,CAAC,MAAM,CAAC,qBAAqB,KAAK,SAAS,IAAI,sBAAsB,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;QACpG,OAAO,OAAO,KAAK,QAAQ;QAC3B,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC;QACzB,OAAO,IAAI,CAAC;QACZ,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ,CACpC,CAAC;AAAA,CACF;AAED,MAAM,UAAU,wBAAwB,CAAC,MAAuB,EAAW;IAC1E,OAAO,qBAAqB,CAAC,MAAM,CAAC,KAAK,SAAS,IAAI,oBAAoB,CAAC,MAAM,CAAC,KAAK,SAAS,CAAC;AAAA,CACjG;AAED,MAAM,UAAU,sBAAsB,CACrC,OAAqC,EACrC,MAAuB,EACvB,EAAE,GAAW,MAAM,CAAC,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EACnB;IAC/B,MAAM,WAAW,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC;IAClD,MAAM,UAAU,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAChD,IAAI,WAAW,KAAK,SAAS,IAAI,UAAU,KAAK,SAAS;QAAE,OAAO,OAAO,CAAC;IAE1E,MAAM,eAAe,GAAG,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC;IAC9C,OAAO;QACN,GAAG,CAAC,WAAW,KAAK,SAAS,IAAI;YAChC,sBAAsB,EAAE,UAAU,CAAC,OAAO,EAAE,sBAAsB,EAAE,WAAW,EAAE,eAAe,CAAC;SACjG,CAAC;QACF,GAAG,CAAC,WAAW,KAAK,SAAS,IAAI,OAAO,EAAE,sBAAsB,KAAK,SAAS;YAC7E,CAAC,CAAC,EAAE,sBAAsB,EAAE,OAAO,CAAC,sBAAsB,EAAE;YAC5D,CAAC,CAAC,EAAE,CAAC;QACN,GAAG,CAAC,UAAU,KAAK,SAAS,IAAI;YAC/B,qBAAqB,EAAE,UAAU,CAAC,OAAO,EAAE,qBAAqB,EAAE,UAAU,EAAE,eAAe,CAAC;SAC9F,CAAC;QACF,GAAG,CAAC,UAAU,KAAK,SAAS,IAAI,OAAO,EAAE,qBAAqB,KAAK,SAAS;YAC3E,CAAC,CAAC,EAAE,qBAAqB,EAAE,OAAO,CAAC,qBAAqB,EAAE;YAC1D,CAAC,CAAC,EAAE,CAAC;QACN,OAAO,EAAE,eAAe,GAAG,CAAC;QAC5B,SAAS,EAAE,EAAE;KACb,CAAC;AAAA,CACF;AAED,MAAM,UAAU,gCAAgC,CAAC,KAA8B,EAA8B;IAC5G,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;IAC9B,MAAM,WAAW,GAAG,OAAO,EAAE,sBAAsB,CAAC;IACpD,IAAI,CAAC,OAAO,IAAI,CAAC,WAAW,IAAI,OAAO,CAAC,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,YAAY,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IAE1F,MAAM,iBAAiB,GAAG,CAAC,KAAK,CAAC,YAAY,GAAG,WAAW,CAAC,GAAG,IAAI,CAAC;IACpE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,iBAAiB,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IAE7E,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,iBAAiB,GAAG,CAAC,CAAC,CAAC,CAAC;IACpF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,CAAC,SAAS,IAAI,uCAAuC,CAAC,CAAC;IAC/G,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAClD,OAAO,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAAA,CACnE;AAED,MAAM,UAAU,2BAA2B,CAAC,OAAgB,EAAU;IACrE,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC;QACjC,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;KACpB,CAAC,CAAC;IACH,IAAI,CAAC,UAAU;QAAE,OAAO,CAAC,CAAC;IAC1B,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,wBAAwB,CAAC,CAAC;AAAA,CAC/D;AAED,MAAM,UAAU,oBAAoB,CAAC,QAAkB,EAAE,QAAwC,EAAY;IAC5G,OAAO,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,EAAE,CAAC;QAC/C,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC;QACzC,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;QACnE,MAAM,kBAAkB,GAAG,KAAK,EAAE,CAAC;QACnC,IAAI,mBAAuC,CAAC;QAC5C,IAAI,cAAkC,CAAC;QACvC,MAAM,kBAAkB,GAAG,aAAa,EAAE,UAAU,CAAC;QACrD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE;YAC5C,GAAG,aAAa;YAChB,UAAU,EAAE,KAAK,EAAE,QAA0B,EAAE,aAAyB,EAAE,EAAE,CAAC;gBAC5E,mBAAmB,KAAK,KAAK,EAAE,CAAC;gBAChC,MAAM,kBAAkB,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;YAAA,CACpD;SACD,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,iCAAiC,EAAE,CAAC;QAElD,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;YACjB,IAAI,cAAc,GAAG,KAAK,CAAC;YAC3B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;gBACjC,IAAI,cAAc,KAAK,SAAS,IAAI,iBAAiB,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC9D,cAAc,GAAG,KAAK,EAAE,CAAC;gBAC1B,CAAC;gBACD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;oBAC3B,cAAc,GAAG,IAAI,CAAC;oBACtB,4BAA4B,CAAC;wBAC5B,gBAAgB,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM;wBAC5C,QAAQ,EAAE,KAAK,EAAE;wBACjB,cAAc;wBACd,KAAK;wBACL,MAAM;wBACN,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK;wBACvC,QAAQ;wBACR,kBAAkB;wBAClB,mBAAmB;qBACnB,CAAC,CAAC;gBACJ,CAAC;qBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;oBACnC,cAAc,GAAG,IAAI,CAAC;gBACvB,CAAC;gBACD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAClB,IAAI,cAAc;oBAAE,OAAO;YAC5B,CAAC;YACD,KAAK,CAAC,GAAG,EAAE,CAAC;QAAA,CACZ,CAAC,EAAE,CAAC;QAEL,OAAO,KAAK,CAAC;IAAA,CACb,CAAC;AAAA,CACF;AAED,SAAS,4BAA4B,CAAC,KAUrC,EAAQ;IACR,IAAI,KAAK,CAAC,cAAc,KAAK,SAAS;QAAE,OAAO;IAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACtD,IAAI,CAAC,QAAQ;QAAE,OAAO;IACtB,MAAM,MAAM,GAAoB;QAC/B,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;QACxC,qBAAqB,EAAE,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,kBAAkB;QACtE,kBAAkB,EAAE,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,cAAc;QACzD,EAAE,EAAE,KAAK,CAAC,MAAM,EAAE;KAClB,CAAC;IACF,IAAI,KAAK,CAAC,mBAAmB,KAAK,SAAS,EAAE,CAAC;QAC7C,MAAM,CAAC,qBAAqB,GAAG,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,mBAAmB,CAAC;IACjF,CAAC;IACD,IAAI,CAAC,wBAAwB,CAAC,MAAM,CAAC;QAAE,OAAO;IAC9C,IAAI,CAAC;QACJ,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACR,gDAAgD;IACjD,CAAC;AAAA,CACD;AAED,SAAS,iBAAiB,CAAC,KAA4B,EAAW;IACjE,OAAO,CACN,CAAC,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,gBAAgB,IAAI,KAAK,CAAC,IAAI,KAAK,gBAAgB,CAAC;QACnG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CACtB,CAAC;AAAA,CACF;AAED,SAAS,qBAAqB,CAAC,MAAuB,EAAsB;IAC3E,MAAM,UAAU,GAAG,MAAM,CAAC,qBAAqB,IAAI,MAAM,CAAC,qBAAqB,CAAC;IAChF,OAAO,eAAe,CAAC,MAAM,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;AAAA,CACxD;AAED,SAAS,oBAAoB,CAAC,MAAuB,EAAsB;IAC1E,OAAO,eAAe,CAAC,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAAC,kBAAkB,CAAC,CAAC;AAAA,CAC3E;AAED,SAAS,eAAe,CAAC,MAA0B,EAAE,UAA8B,EAAsB;IACxG,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,CAAC,UAAU,CAAC;QAAE,OAAO,SAAS,CAAC;IAC7F,OAAO,MAAM,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;AAAA,CACpC;AAED,SAAS,UAAU,CAAC,QAA4B,EAAE,IAAY,EAAE,eAAuB,EAAU;IAChG,IAAI,QAAQ,KAAK,SAAS,IAAI,eAAe,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACjE,OAAO,QAAQ,GAAG,CAAC,CAAC,GAAG,eAAe,CAAC,GAAG,IAAI,GAAG,eAAe,CAAC;AAAA,CACjE;AAED,SAAS,sBAAsB,CAAC,KAAc,EAAmB;IAChE,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC;AAAA,CACxE","sourcesContent":["import type { StreamFn, StreamIdleOptions } from \"@caupulican/pi-agent-core\";\nimport {\n\ttype Api,\n\ttype AssistantMessageEvent,\n\ttype Context,\n\tcreateAssistantMessageEventStream,\n\ttype Model,\n\ttype ProviderResponse,\n} from \"@caupulican/pi-ai\";\n\nconst PERF_EWMA_ALPHA = 0.3;\nconst CHARS_PER_TOKEN_ESTIMATE = 4;\nexport const DEFAULT_ADAPTIVE_STREAM_IDLE_CEILING_MS = 30 * 60 * 1000;\n\nexport interface ModelPerfProfile {\n\tprefillTokensPerSecond?: number;\n\tdecodeTokensPerSecond?: number;\n\tsamples: number;\n\tupdatedAt: string;\n}\n\nexport interface ModelPerfSample {\n\tpromptTokens?: number;\n\tcompletionTokens?: number;\n\theadersToFirstTokenMs?: number;\n\trequestToFirstTokenMs?: number;\n\tfirstTokenToDoneMs?: number;\n\tat?: string;\n}\n\nexport interface AdaptiveStreamIdleInput {\n\tbase: StreamIdleOptions;\n\tprofile?: ModelPerfProfile;\n\tpromptTokens: number;\n\tceilingMs?: number;\n}\n\nexport interface ModelPerfProfileStreamRecorder {\n\tmodelKey: (model: Model<Api>) => string | undefined;\n\trecordSample: (modelKey: string, sample: ModelPerfSample) => void;\n\tnowMs?: () => number;\n\tnowIso?: () => string;\n}\n\nexport function isModelPerfProfile(value: unknown): value is ModelPerfProfile {\n\tif (typeof value !== \"object\" || value === null || Array.isArray(value)) return false;\n\tconst record = value as Record<string, unknown>;\n\tconst samples = record.samples;\n\treturn (\n\t\t(record.prefillTokensPerSecond === undefined || isPositiveFiniteNumber(record.prefillTokensPerSecond)) &&\n\t\t(record.decodeTokensPerSecond === undefined || isPositiveFiniteNumber(record.decodeTokensPerSecond)) &&\n\t\ttypeof samples === \"number\" &&\n\t\tNumber.isInteger(samples) &&\n\t\tsamples >= 0 &&\n\t\ttypeof record.updatedAt === \"string\"\n\t);\n}\n\nexport function hasUsableModelPerfSample(sample: ModelPerfSample): boolean {\n\treturn prefillRateFromSample(sample) !== undefined || decodeRateFromSample(sample) !== undefined;\n}\n\nexport function updateModelPerfProfile(\n\tcurrent: ModelPerfProfile | undefined,\n\tsample: ModelPerfSample,\n\tat: string = sample.at ?? new Date().toISOString(),\n): ModelPerfProfile | undefined {\n\tconst prefillRate = prefillRateFromSample(sample);\n\tconst decodeRate = decodeRateFromSample(sample);\n\tif (prefillRate === undefined && decodeRate === undefined) return current;\n\n\tconst previousSamples = current?.samples ?? 0;\n\treturn {\n\t\t...(prefillRate !== undefined && {\n\t\t\tprefillTokensPerSecond: updateEwma(current?.prefillTokensPerSecond, prefillRate, previousSamples),\n\t\t}),\n\t\t...(prefillRate === undefined && current?.prefillTokensPerSecond !== undefined\n\t\t\t? { prefillTokensPerSecond: current.prefillTokensPerSecond }\n\t\t\t: {}),\n\t\t...(decodeRate !== undefined && {\n\t\t\tdecodeTokensPerSecond: updateEwma(current?.decodeTokensPerSecond, decodeRate, previousSamples),\n\t\t}),\n\t\t...(decodeRate === undefined && current?.decodeTokensPerSecond !== undefined\n\t\t\t? { decodeTokensPerSecond: current.decodeTokensPerSecond }\n\t\t\t: {}),\n\t\tsamples: previousSamples + 1,\n\t\tupdatedAt: at,\n\t};\n}\n\nexport function resolveAdaptiveStreamIdleOptions(input: AdaptiveStreamIdleInput): Partial<StreamIdleOptions> {\n\tconst profile = input.profile;\n\tconst prefillRate = profile?.prefillTokensPerSecond;\n\tif (!profile || !prefillRate || profile.samples < 1 || input.promptTokens <= 0) return {};\n\n\tconst expectedPrefillMs = (input.promptTokens / prefillRate) * 1000;\n\tif (!Number.isFinite(expectedPrefillMs) || expectedPrefillMs <= 0) return {};\n\n\tconst targetMs = Math.ceil(Math.max(input.base.quietIdleMs, expectedPrefillMs * 3));\n\tconst ceilingMs = Math.max(input.base.quietIdleMs, input.ceilingMs ?? DEFAULT_ADAPTIVE_STREAM_IDLE_CEILING_MS);\n\tconst quietIdleMs = Math.min(targetMs, ceilingMs);\n\treturn quietIdleMs > input.base.quietIdleMs ? { quietIdleMs } : {};\n}\n\nexport function estimateContextPromptTokens(context: Context): number {\n\tconst serialized = JSON.stringify({\n\t\tsystemPrompt: context.systemPrompt,\n\t\tmessages: context.messages,\n\t\ttools: context.tools,\n\t});\n\tif (!serialized) return 0;\n\treturn Math.ceil(serialized.length / CHARS_PER_TOKEN_ESTIMATE);\n}\n\nexport function withModelPerfProfile(streamFn: StreamFn, recorder: ModelPerfProfileStreamRecorder): StreamFn {\n\treturn async (model, context, streamOptions) => {\n\t\tconst nowMs = recorder.nowMs ?? Date.now;\n\t\tconst nowIso = recorder.nowIso ?? (() => new Date().toISOString());\n\t\tconst requestStartedAtMs = nowMs();\n\t\tlet responseHeadersAtMs: number | undefined;\n\t\tlet firstTokenAtMs: number | undefined;\n\t\tconst originalOnResponse = streamOptions?.onResponse;\n\t\tconst inner = await streamFn(model, context, {\n\t\t\t...streamOptions,\n\t\t\tonResponse: async (response: ProviderResponse, responseModel: Model<Api>) => {\n\t\t\t\tresponseHeadersAtMs ??= nowMs();\n\t\t\t\tawait originalOnResponse?.(response, responseModel);\n\t\t\t},\n\t\t});\n\t\tconst outer = createAssistantMessageEventStream();\n\n\t\tvoid (async () => {\n\t\t\tlet terminalPushed = false;\n\t\t\tfor await (const event of inner) {\n\t\t\t\tif (firstTokenAtMs === undefined && isFirstTokenEvent(event)) {\n\t\t\t\t\tfirstTokenAtMs = nowMs();\n\t\t\t\t}\n\t\t\t\tif (event.type === \"done\") {\n\t\t\t\t\tterminalPushed = true;\n\t\t\t\t\trecordSuccessfulStreamSample({\n\t\t\t\t\t\tcompletionTokens: event.message.usage.output,\n\t\t\t\t\t\tdoneAtMs: nowMs(),\n\t\t\t\t\t\tfirstTokenAtMs,\n\t\t\t\t\t\tmodel,\n\t\t\t\t\t\tnowIso,\n\t\t\t\t\t\tpromptTokens: event.message.usage.input,\n\t\t\t\t\t\trecorder,\n\t\t\t\t\t\trequestStartedAtMs,\n\t\t\t\t\t\tresponseHeadersAtMs,\n\t\t\t\t\t});\n\t\t\t\t} else if (event.type === \"error\") {\n\t\t\t\t\tterminalPushed = true;\n\t\t\t\t}\n\t\t\t\touter.push(event);\n\t\t\t\tif (terminalPushed) return;\n\t\t\t}\n\t\t\touter.end();\n\t\t})();\n\n\t\treturn outer;\n\t};\n}\n\nfunction recordSuccessfulStreamSample(input: {\n\tmodel: Model<Api>;\n\trecorder: ModelPerfProfileStreamRecorder;\n\trequestStartedAtMs: number;\n\tresponseHeadersAtMs: number | undefined;\n\tfirstTokenAtMs: number | undefined;\n\tdoneAtMs: number;\n\tpromptTokens: number;\n\tcompletionTokens: number;\n\tnowIso: () => string;\n}): void {\n\tif (input.firstTokenAtMs === undefined) return;\n\tconst modelKey = input.recorder.modelKey(input.model);\n\tif (!modelKey) return;\n\tconst sample: ModelPerfSample = {\n\t\tpromptTokens: input.promptTokens,\n\t\tcompletionTokens: input.completionTokens,\n\t\trequestToFirstTokenMs: input.firstTokenAtMs - input.requestStartedAtMs,\n\t\tfirstTokenToDoneMs: input.doneAtMs - input.firstTokenAtMs,\n\t\tat: input.nowIso(),\n\t};\n\tif (input.responseHeadersAtMs !== undefined) {\n\t\tsample.headersToFirstTokenMs = input.firstTokenAtMs - input.responseHeadersAtMs;\n\t}\n\tif (!hasUsableModelPerfSample(sample)) return;\n\ttry {\n\t\tinput.recorder.recordSample(modelKey, sample);\n\t} catch {\n\t\t// Perf profiling must never fail the user turn.\n\t}\n}\n\nfunction isFirstTokenEvent(event: AssistantMessageEvent): boolean {\n\treturn (\n\t\t(event.type === \"text_delta\" || event.type === \"thinking_delta\" || event.type === \"toolcall_delta\") &&\n\t\tevent.delta.length > 0\n\t);\n}\n\nfunction prefillRateFromSample(sample: ModelPerfSample): number | undefined {\n\tconst durationMs = sample.headersToFirstTokenMs ?? sample.requestToFirstTokenMs;\n\treturn tokensPerSecond(sample.promptTokens, durationMs);\n}\n\nfunction decodeRateFromSample(sample: ModelPerfSample): number | undefined {\n\treturn tokensPerSecond(sample.completionTokens, sample.firstTokenToDoneMs);\n}\n\nfunction tokensPerSecond(tokens: number | undefined, durationMs: number | undefined): number | undefined {\n\tif (!isPositiveFiniteNumber(tokens) || !isPositiveFiniteNumber(durationMs)) return undefined;\n\treturn tokens / (durationMs / 1000);\n}\n\nfunction updateEwma(previous: number | undefined, next: number, previousSamples: number): number {\n\tif (previous === undefined || previousSamples === 0) return next;\n\treturn previous * (1 - PERF_EWMA_ALPHA) + next * PERF_EWMA_ALPHA;\n}\n\nfunction isPositiveFiniteNumber(value: unknown): value is number {\n\treturn typeof value === \"number\" && Number.isFinite(value) && value > 0;\n}\n"]}
+{"version":3,"file":"perf-profile.js","sourceRoot":"","sources":["../../../src/core/models/perf-profile.ts"],"names":[],"mappings":"AACA,OAAO,EAIN,iCAAiC,GAGjC,MAAM,mBAAmB,CAAC;AAE3B,MAAM,eAAe,GAAG,GAAG,CAAC;AAC5B,MAAM,wBAAwB,GAAG,CAAC,CAAC;AACnC,MAAM,CAAC,MAAM,uCAAuC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAmCtE,MAAM,UAAU,kBAAkB,CAAC,KAAc,EAA6B;IAC7E,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACtF,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;IAC/B,OAAO,CACN,CAAC,MAAM,CAAC,sBAAsB,KAAK,SAAS,IAAI,sBAAsB,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;QACtG,CAAC,MAAM,CAAC,qBAAqB,KAAK,SAAS,IAAI,sBAAsB,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;QACpG,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,sBAAsB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACtE,OAAO,OAAO,KAAK,QAAQ;QAC3B,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC;QACzB,OAAO,IAAI,CAAC;QACZ,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ,CACpC,CAAC;AAAA,CACF;AAED,MAAM,UAAU,wBAAwB,CAAC,MAAuB,EAAW;IAC1E,OAAO,CACN,qBAAqB,CAAC,MAAM,CAAC,KAAK,SAAS;QAC3C,oBAAoB,CAAC,MAAM,CAAC,KAAK,SAAS;QAC1C,sBAAsB,CAAC,MAAM,CAAC,MAAM,CAAC,CACrC,CAAC;AAAA,CACF;AAED,MAAM,UAAU,sBAAsB,CACrC,OAAqC,EACrC,MAAuB,EACvB,EAAE,GAAW,MAAM,CAAC,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EACnB;IAC/B,MAAM,WAAW,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC;IAClD,MAAM,UAAU,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,sBAAsB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACjF,IAAI,WAAW,KAAK,SAAS,IAAI,UAAU,KAAK,SAAS,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,OAAO,CAAC;IAElG,MAAM,eAAe,GAAG,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC;IAC9C,OAAO;QACN,GAAG,CAAC,WAAW,KAAK,SAAS,IAAI;YAChC,sBAAsB,EAAE,UAAU,CAAC,OAAO,EAAE,sBAAsB,EAAE,WAAW,EAAE,eAAe,CAAC;SACjG,CAAC;QACF,GAAG,CAAC,WAAW,KAAK,SAAS,IAAI,OAAO,EAAE,sBAAsB,KAAK,SAAS;YAC7E,CAAC,CAAC,EAAE,sBAAsB,EAAE,OAAO,CAAC,sBAAsB,EAAE;YAC5D,CAAC,CAAC,EAAE,CAAC;QACN,GAAG,CAAC,UAAU,KAAK,SAAS,IAAI;YAC/B,qBAAqB,EAAE,UAAU,CAAC,OAAO,EAAE,qBAAqB,EAAE,UAAU,EAAE,eAAe,CAAC;SAC9F,CAAC;QACF,GAAG,CAAC,UAAU,KAAK,SAAS,IAAI,OAAO,EAAE,qBAAqB,KAAK,SAAS;YAC3E,CAAC,CAAC,EAAE,qBAAqB,EAAE,OAAO,CAAC,qBAAqB,EAAE;YAC1D,CAAC,CAAC,EAAE,CAAC;QACN,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,CAAC,EAAE,CAAC;QAC7F,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,OAAO,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5F,OAAO,EAAE,eAAe,GAAG,CAAC;QAC5B,SAAS,EAAE,EAAE;KACb,CAAC;AAAA,CACF;AAED,MAAM,UAAU,gCAAgC,CAAC,KAA8B,EAA8B;IAC5G,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;IAC9B,MAAM,MAAM,GAA+B,EAAE,CAAC;IAC9C,MAAM,qBAAqB,GAAG,KAAK,CAAC,UAAU;QAC7C,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC;QACxD,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC;IACxB,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,CAAC,SAAS,IAAI,uCAAuC,CAAC,CAAC;IACpH,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,KAAK,CAAC,SAAS,IAAI,uCAAuC,CAAC,CAAC;IAErH,MAAM,iBAAiB,GAAG,0BAA0B,CAAC,OAAO,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAClF,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;QACrC,MAAM,WAAW,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,cAAc,EAAE,iBAAiB,CAAC,CAAC;QAC7F,IAAI,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW;YAAE,MAAM,CAAC,WAAW,GAAG,WAAW,CAAC;IAC5E,CAAC;IAED,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;QACtB,MAAM,qBAAqB,GAC1B,iBAAiB,KAAK,SAAS,CAAC,CAAC,CAAC,iBAAiB,GAAG,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC;QAChG,MAAM,SAAS,GACd,qBAAqB,KAAK,SAAS;YAClC,CAAC,CAAC,aAAa,CAAC,qBAAqB,EAAE,gBAAgB,EAAE,qBAAqB,CAAC;YAC/E,CAAC,CAAC,qBAAqB,CAAC;QAC1B,IAAI,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS;YAAE,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC;IACpE,CAAC;IAED,OAAO,MAAM,CAAC;AAAA,CACd;AAED,MAAM,UAAU,2BAA2B,CAAC,OAAgB,EAAU;IACrE,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC;QACjC,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;KACpB,CAAC,CAAC;IACH,IAAI,CAAC,UAAU;QAAE,OAAO,CAAC,CAAC;IAC1B,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,wBAAwB,CAAC,CAAC;AAAA,CAC/D;AAED,MAAM,UAAU,oBAAoB,CAAC,QAAkB,EAAE,QAAwC,EAAY;IAC5G,OAAO,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,EAAE,CAAC;QAC/C,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC;QACzC,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;QACnE,MAAM,kBAAkB,GAAG,KAAK,EAAE,CAAC;QACnC,IAAI,mBAAuC,CAAC;QAC5C,IAAI,cAAkC,CAAC;QACvC,MAAM,kBAAkB,GAAG,aAAa,EAAE,UAAU,CAAC;QACrD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE;YAC5C,GAAG,aAAa;YAChB,UAAU,EAAE,KAAK,EAAE,QAA0B,EAAE,aAAyB,EAAE,EAAE,CAAC;gBAC5E,mBAAmB,KAAK,KAAK,EAAE,CAAC;gBAChC,MAAM,kBAAkB,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;YAAA,CACpD;SACD,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,iCAAiC,EAAE,CAAC;QAElD,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;YACjB,IAAI,cAAc,GAAG,KAAK,CAAC;YAC3B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;gBACjC,IAAI,cAAc,KAAK,SAAS,IAAI,iBAAiB,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC9D,cAAc,GAAG,KAAK,EAAE,CAAC;gBAC1B,CAAC;gBACD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;oBAC3B,cAAc,GAAG,IAAI,CAAC;oBACtB,4BAA4B,CAAC;wBAC5B,gBAAgB,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM;wBAC5C,QAAQ,EAAE,KAAK,EAAE;wBACjB,cAAc;wBACd,KAAK;wBACL,MAAM;wBACN,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK;wBACvC,QAAQ;wBACR,kBAAkB;wBAClB,mBAAmB;qBACnB,CAAC,CAAC;gBACJ,CAAC;qBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;oBACnC,cAAc,GAAG,IAAI,CAAC;gBACvB,CAAC;gBACD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAClB,IAAI,cAAc;oBAAE,OAAO;YAC5B,CAAC;YACD,KAAK,CAAC,GAAG,EAAE,CAAC;QAAA,CACZ,CAAC,EAAE,CAAC;QAEL,OAAO,KAAK,CAAC;IAAA,CACb,CAAC;AAAA,CACF;AAED,SAAS,4BAA4B,CAAC,KAUrC,EAAQ;IACR,IAAI,KAAK,CAAC,cAAc,KAAK,SAAS;QAAE,OAAO;IAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACtD,IAAI,CAAC,QAAQ;QAAE,OAAO;IACtB,MAAM,MAAM,GAAoB;QAC/B,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;QACxC,qBAAqB,EAAE,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,kBAAkB;QACtE,kBAAkB,EAAE,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,cAAc;QACzD,EAAE,EAAE,KAAK,CAAC,MAAM,EAAE;KAClB,CAAC;IACF,IAAI,KAAK,CAAC,mBAAmB,KAAK,SAAS,EAAE,CAAC;QAC7C,MAAM,CAAC,qBAAqB,GAAG,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,mBAAmB,CAAC;IACjF,CAAC;IACD,IAAI,CAAC,wBAAwB,CAAC,MAAM,CAAC;QAAE,OAAO;IAC9C,IAAI,CAAC;QACJ,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACR,gDAAgD;IACjD,CAAC;AAAA,CACD;AAED,SAAS,iBAAiB,CAAC,KAA4B,EAAW;IACjE,OAAO,CACN,CAAC,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,gBAAgB,IAAI,KAAK,CAAC,IAAI,KAAK,gBAAgB,CAAC;QACnG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CACtB,CAAC;AAAA,CACF;AAED,SAAS,0BAA0B,CAAC,OAAqC,EAAE,YAAoB,EAAsB;IACpH,MAAM,WAAW,GAAG,OAAO,EAAE,sBAAsB,CAAC;IACpD,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,YAAY,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IAC3F,MAAM,iBAAiB,GAAG,CAAC,YAAY,GAAG,WAAW,CAAC,GAAG,IAAI,CAAC;IAC9D,OAAO,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,iBAAiB,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,SAAS,CAAC;AAAA,CACnG;AAED,SAAS,aAAa,CAAC,SAAiB,EAAE,SAAiB,EAAE,UAAkB,EAAU;IACxF,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;AAAA,CAC3E;AAED,SAAS,qBAAqB,CAAC,MAAuB,EAAsB;IAC3E,MAAM,UAAU,GAAG,sBAAsB,CAAC,MAAM,CAAC,qBAAqB,CAAC;QACtE,CAAC,CAAC,MAAM,CAAC,qBAAqB;QAC9B,CAAC,CAAC,MAAM,CAAC,qBAAqB,CAAC;IAChC,OAAO,eAAe,CAAC,MAAM,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;AAAA,CACxD;AAED,SAAS,oBAAoB,CAAC,MAAuB,EAAsB;IAC1E,OAAO,eAAe,CAAC,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAAC,kBAAkB,CAAC,CAAC;AAAA,CAC3E;AAED,SAAS,eAAe,CAAC,MAA0B,EAAE,UAA8B,EAAsB;IACxG,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,CAAC,UAAU,CAAC;QAAE,OAAO,SAAS,CAAC;IAC7F,OAAO,MAAM,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;AAAA,CACpC;AAED,SAAS,UAAU,CAAC,QAA4B,EAAE,IAAY,EAAE,eAAuB,EAAU;IAChG,IAAI,QAAQ,KAAK,SAAS,IAAI,eAAe,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACjE,OAAO,QAAQ,GAAG,CAAC,CAAC,GAAG,eAAe,CAAC,GAAG,IAAI,GAAG,eAAe,CAAC;AAAA,CACjE;AAED,SAAS,sBAAsB,CAAC,KAAc,EAAmB;IAChE,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC;AAAA,CACxE","sourcesContent":["import type { StreamFn, StreamIdleOptions } from \"@caupulican/pi-agent-core\";\nimport {\n\ttype Api,\n\ttype AssistantMessageEvent,\n\ttype Context,\n\tcreateAssistantMessageEventStream,\n\ttype Model,\n\ttype ProviderResponse,\n} from \"@caupulican/pi-ai\";\n\nconst PERF_EWMA_ALPHA = 0.3;\nconst CHARS_PER_TOKEN_ESTIMATE = 4;\nexport const DEFAULT_ADAPTIVE_STREAM_IDLE_CEILING_MS = 30 * 60 * 1000;\n\nexport interface ModelPerfProfile {\n\tprefillTokensPerSecond?: number;\n\tdecodeTokensPerSecond?: number;\n\tloadMs?: number;\n\tsamples: number;\n\tupdatedAt: string;\n}\n\nexport interface ModelPerfSample {\n\tpromptTokens?: number;\n\tcompletionTokens?: number;\n\theadersToFirstTokenMs?: number;\n\trequestToFirstTokenMs?: number;\n\tfirstTokenToDoneMs?: number;\n\tloadMs?: number;\n\tat?: string;\n}\n\nexport interface AdaptiveStreamIdleInput {\n\tbase: StreamIdleOptions;\n\tprofile?: ModelPerfProfile;\n\tpromptTokens: number;\n\tceilingMs?: number;\n\tlocalClass?: boolean;\n}\n\nexport interface ModelPerfProfileStreamRecorder {\n\tmodelKey: (model: Model<Api>) => string | undefined;\n\trecordSample: (modelKey: string, sample: ModelPerfSample) => void;\n\tnowMs?: () => number;\n\tnowIso?: () => string;\n}\n\nexport function isModelPerfProfile(value: unknown): value is ModelPerfProfile {\n\tif (typeof value !== \"object\" || value === null || Array.isArray(value)) return false;\n\tconst record = value as Record<string, unknown>;\n\tconst samples = record.samples;\n\treturn (\n\t\t(record.prefillTokensPerSecond === undefined || isPositiveFiniteNumber(record.prefillTokensPerSecond)) &&\n\t\t(record.decodeTokensPerSecond === undefined || isPositiveFiniteNumber(record.decodeTokensPerSecond)) &&\n\t\t(record.loadMs === undefined || isPositiveFiniteNumber(record.loadMs)) &&\n\t\ttypeof samples === \"number\" &&\n\t\tNumber.isInteger(samples) &&\n\t\tsamples >= 0 &&\n\t\ttypeof record.updatedAt === \"string\"\n\t);\n}\n\nexport function hasUsableModelPerfSample(sample: ModelPerfSample): boolean {\n\treturn (\n\t\tprefillRateFromSample(sample) !== undefined ||\n\t\tdecodeRateFromSample(sample) !== undefined ||\n\t\tisPositiveFiniteNumber(sample.loadMs)\n\t);\n}\n\nexport function updateModelPerfProfile(\n\tcurrent: ModelPerfProfile | undefined,\n\tsample: ModelPerfSample,\n\tat: string = sample.at ?? new Date().toISOString(),\n): ModelPerfProfile | undefined {\n\tconst prefillRate = prefillRateFromSample(sample);\n\tconst decodeRate = decodeRateFromSample(sample);\n\tconst loadMs = isPositiveFiniteNumber(sample.loadMs) ? sample.loadMs : undefined;\n\tif (prefillRate === undefined && decodeRate === undefined && loadMs === undefined) return current;\n\n\tconst previousSamples = current?.samples ?? 0;\n\treturn {\n\t\t...(prefillRate !== undefined && {\n\t\t\tprefillTokensPerSecond: updateEwma(current?.prefillTokensPerSecond, prefillRate, previousSamples),\n\t\t}),\n\t\t...(prefillRate === undefined && current?.prefillTokensPerSecond !== undefined\n\t\t\t? { prefillTokensPerSecond: current.prefillTokensPerSecond }\n\t\t\t: {}),\n\t\t...(decodeRate !== undefined && {\n\t\t\tdecodeTokensPerSecond: updateEwma(current?.decodeTokensPerSecond, decodeRate, previousSamples),\n\t\t}),\n\t\t...(decodeRate === undefined && current?.decodeTokensPerSecond !== undefined\n\t\t\t? { decodeTokensPerSecond: current.decodeTokensPerSecond }\n\t\t\t: {}),\n\t\t...(loadMs !== undefined && { loadMs: updateEwma(current?.loadMs, loadMs, previousSamples) }),\n\t\t...(loadMs === undefined && current?.loadMs !== undefined ? { loadMs: current.loadMs } : {}),\n\t\tsamples: previousSamples + 1,\n\t\tupdatedAt: at,\n\t};\n}\n\nexport function resolveAdaptiveStreamIdleOptions(input: AdaptiveStreamIdleInput): Partial<StreamIdleOptions> {\n\tconst profile = input.profile;\n\tconst result: Partial<StreamIdleOptions> = {};\n\tconst localConnectDefaultMs = input.localClass\n\t\t? Math.max(input.base.connectMs, input.base.quietIdleMs)\n\t\t: input.base.connectMs;\n\tconst quietCeilingMs = Math.max(input.base.quietIdleMs, input.ceilingMs ?? DEFAULT_ADAPTIVE_STREAM_IDLE_CEILING_MS);\n\tconst connectCeilingMs = Math.max(localConnectDefaultMs, input.ceilingMs ?? DEFAULT_ADAPTIVE_STREAM_IDLE_CEILING_MS);\n\n\tconst expectedPrefillMs = expectedPrefillFromProfile(profile, input.promptTokens);\n\tif (expectedPrefillMs !== undefined) {\n\t\tconst quietIdleMs = adaptiveBound(input.base.quietIdleMs, quietCeilingMs, expectedPrefillMs);\n\t\tif (quietIdleMs > input.base.quietIdleMs) result.quietIdleMs = quietIdleMs;\n\t}\n\n\tif (input.localClass) {\n\t\tconst expectedConnectWorkMs =\n\t\t\texpectedPrefillMs !== undefined ? expectedPrefillMs + (profile?.loadMs ?? 0) : profile?.loadMs;\n\t\tconst connectMs =\n\t\t\texpectedConnectWorkMs !== undefined\n\t\t\t\t? adaptiveBound(localConnectDefaultMs, connectCeilingMs, expectedConnectWorkMs)\n\t\t\t\t: localConnectDefaultMs;\n\t\tif (connectMs > input.base.connectMs) result.connectMs = connectMs;\n\t}\n\n\treturn result;\n}\n\nexport function estimateContextPromptTokens(context: Context): number {\n\tconst serialized = JSON.stringify({\n\t\tsystemPrompt: context.systemPrompt,\n\t\tmessages: context.messages,\n\t\ttools: context.tools,\n\t});\n\tif (!serialized) return 0;\n\treturn Math.ceil(serialized.length / CHARS_PER_TOKEN_ESTIMATE);\n}\n\nexport function withModelPerfProfile(streamFn: StreamFn, recorder: ModelPerfProfileStreamRecorder): StreamFn {\n\treturn async (model, context, streamOptions) => {\n\t\tconst nowMs = recorder.nowMs ?? Date.now;\n\t\tconst nowIso = recorder.nowIso ?? (() => new Date().toISOString());\n\t\tconst requestStartedAtMs = nowMs();\n\t\tlet responseHeadersAtMs: number | undefined;\n\t\tlet firstTokenAtMs: number | undefined;\n\t\tconst originalOnResponse = streamOptions?.onResponse;\n\t\tconst inner = await streamFn(model, context, {\n\t\t\t...streamOptions,\n\t\t\tonResponse: async (response: ProviderResponse, responseModel: Model<Api>) => {\n\t\t\t\tresponseHeadersAtMs ??= nowMs();\n\t\t\t\tawait originalOnResponse?.(response, responseModel);\n\t\t\t},\n\t\t});\n\t\tconst outer = createAssistantMessageEventStream();\n\n\t\tvoid (async () => {\n\t\t\tlet terminalPushed = false;\n\t\t\tfor await (const event of inner) {\n\t\t\t\tif (firstTokenAtMs === undefined && isFirstTokenEvent(event)) {\n\t\t\t\t\tfirstTokenAtMs = nowMs();\n\t\t\t\t}\n\t\t\t\tif (event.type === \"done\") {\n\t\t\t\t\tterminalPushed = true;\n\t\t\t\t\trecordSuccessfulStreamSample({\n\t\t\t\t\t\tcompletionTokens: event.message.usage.output,\n\t\t\t\t\t\tdoneAtMs: nowMs(),\n\t\t\t\t\t\tfirstTokenAtMs,\n\t\t\t\t\t\tmodel,\n\t\t\t\t\t\tnowIso,\n\t\t\t\t\t\tpromptTokens: event.message.usage.input,\n\t\t\t\t\t\trecorder,\n\t\t\t\t\t\trequestStartedAtMs,\n\t\t\t\t\t\tresponseHeadersAtMs,\n\t\t\t\t\t});\n\t\t\t\t} else if (event.type === \"error\") {\n\t\t\t\t\tterminalPushed = true;\n\t\t\t\t}\n\t\t\t\touter.push(event);\n\t\t\t\tif (terminalPushed) return;\n\t\t\t}\n\t\t\touter.end();\n\t\t})();\n\n\t\treturn outer;\n\t};\n}\n\nfunction recordSuccessfulStreamSample(input: {\n\tmodel: Model<Api>;\n\trecorder: ModelPerfProfileStreamRecorder;\n\trequestStartedAtMs: number;\n\tresponseHeadersAtMs: number | undefined;\n\tfirstTokenAtMs: number | undefined;\n\tdoneAtMs: number;\n\tpromptTokens: number;\n\tcompletionTokens: number;\n\tnowIso: () => string;\n}): void {\n\tif (input.firstTokenAtMs === undefined) return;\n\tconst modelKey = input.recorder.modelKey(input.model);\n\tif (!modelKey) return;\n\tconst sample: ModelPerfSample = {\n\t\tpromptTokens: input.promptTokens,\n\t\tcompletionTokens: input.completionTokens,\n\t\trequestToFirstTokenMs: input.firstTokenAtMs - input.requestStartedAtMs,\n\t\tfirstTokenToDoneMs: input.doneAtMs - input.firstTokenAtMs,\n\t\tat: input.nowIso(),\n\t};\n\tif (input.responseHeadersAtMs !== undefined) {\n\t\tsample.headersToFirstTokenMs = input.firstTokenAtMs - input.responseHeadersAtMs;\n\t}\n\tif (!hasUsableModelPerfSample(sample)) return;\n\ttry {\n\t\tinput.recorder.recordSample(modelKey, sample);\n\t} catch {\n\t\t// Perf profiling must never fail the user turn.\n\t}\n}\n\nfunction isFirstTokenEvent(event: AssistantMessageEvent): boolean {\n\treturn (\n\t\t(event.type === \"text_delta\" || event.type === \"thinking_delta\" || event.type === \"toolcall_delta\") &&\n\t\tevent.delta.length > 0\n\t);\n}\n\nfunction expectedPrefillFromProfile(profile: ModelPerfProfile | undefined, promptTokens: number): number | undefined {\n\tconst prefillRate = profile?.prefillTokensPerSecond;\n\tif (!profile || profile.samples < 1 || !prefillRate || promptTokens <= 0) return undefined;\n\tconst expectedPrefillMs = (promptTokens / prefillRate) * 1000;\n\treturn Number.isFinite(expectedPrefillMs) && expectedPrefillMs > 0 ? expectedPrefillMs : undefined;\n}\n\nfunction adaptiveBound(defaultMs: number, ceilingMs: number, expectedMs: number): number {\n\treturn Math.min(Math.ceil(Math.max(defaultMs, expectedMs * 3)), ceilingMs);\n}\n\nfunction prefillRateFromSample(sample: ModelPerfSample): number | undefined {\n\tconst durationMs = isPositiveFiniteNumber(sample.headersToFirstTokenMs)\n\t\t? sample.headersToFirstTokenMs\n\t\t: sample.requestToFirstTokenMs;\n\treturn tokensPerSecond(sample.promptTokens, durationMs);\n}\n\nfunction decodeRateFromSample(sample: ModelPerfSample): number | undefined {\n\treturn tokensPerSecond(sample.completionTokens, sample.firstTokenToDoneMs);\n}\n\nfunction tokensPerSecond(tokens: number | undefined, durationMs: number | undefined): number | undefined {\n\tif (!isPositiveFiniteNumber(tokens) || !isPositiveFiniteNumber(durationMs)) return undefined;\n\treturn tokens / (durationMs / 1000);\n}\n\nfunction updateEwma(previous: number | undefined, next: number, previousSamples: number): number {\n\tif (previous === undefined || previousSamples === 0) return next;\n\treturn previous * (1 - PERF_EWMA_ALPHA) + next * PERF_EWMA_ALPHA;\n}\n\nfunction isPositiveFiniteNumber(value: unknown): value is number {\n\treturn typeof value === \"number\" && Number.isFinite(value) && value > 0;\n}\n"]}

package/dist/core/models/runtime-arbiter.d.ts

@@ -1,3 +1,4 @@
+import type { OllamaRuntime, TransformersRuntime } from "./local-runtime.ts";
 export type ResidencyControl = "full" | "keep-alive" | "advisory";
 export type RuntimeRole = "active" | "router" | "bench" | "probe";
 export interface RuntimeResidentModel {
@@ -49,6 +50,50 @@ export interface RuntimeResidencyAdapter {
     ensureResident(model: string): Promise<void>;
     release(model: string): Promise<void>;
 }
+export declare class RuntimeResidencyArbiter {
+    private readonly budgetBytes;
+    private readonly adapters;
+    constructor(args: {
+        budgetBytes: number;
+        adapters: RuntimeResidencyAdapter[];
+    });
+    ensureResident(adapterId: string, request: RuntimeLoadRequest): Promise<RuntimeResidencyPlan>;
+}
+export declare class OllamaRuntimeResidencyAdapter implements RuntimeResidencyAdapter {
+    readonly id: string;
+    readonly residencyControl: "keep-alive";
+    private readonly runtime;
+    private readonly nowMs;
+    constructor(id: string, runtime: Pick<OllamaRuntime, "listResidentModels" | "ensureResident" | "releaseResident">, options?: {
+        nowMs?: () => number;
+    });
+    list(): Promise<RuntimeResidentModel[]>;
+    ensureResident(model: string): Promise<void>;
+    release(model: string): Promise<void>;
+}
+export declare class TransformersRuntimeResidencyAdapter implements RuntimeResidencyAdapter {
+    readonly id: string;
+    readonly residencyControl: "full";
+    private readonly runtime;
+    private readonly model;
+    private readonly bytes;
+    private readonly nowMs;
+    constructor(id: string, runtime: Pick<TransformersRuntime, "detect" | "start" | "stop">, model: string, bytes: number, options?: {
+        nowMs?: () => number;
+    });
+    list(): Promise<RuntimeResidentModel[]>;
+    ensureResident(_model: string): Promise<void>;
+    release(_model: string): Promise<void>;
+}
+export declare class AdvisoryRuntimeResidencyAdapter implements RuntimeResidencyAdapter {
+    readonly id: string;
+    readonly residencyControl: "advisory";
+    private readonly residents;
+    constructor(id: string, residents: Omit<RuntimeResidentModel, "adapterId" | "residencyControl">[]);
+    list(): Promise<RuntimeResidentModel[]>;
+    ensureResident(_model: string): Promise<void>;
+    release(_model: string): Promise<void>;
+}
 export declare function planRuntimeResidency(args: {
     budgetBytes: number;
     residents: readonly RuntimeResidentModel[];

package/dist/core/models/runtime-arbiter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runtime-arbiter.d.ts","sourceRoot":"","sources":["../../../src/core/models/runtime-arbiter.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,gBAAgB,GAAG,MAAM,GAAG,YAAY,GAAG,UAAU,CAAC;AAClE,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,CAAC;AAElE,MAAM,WAAW,oBAAoB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,gBAAgB,EAAE,gBAAgB,CAAC;CACnC;AAED,MAAM,WAAW,kBAAkB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,kBAAkB,EAAE,CAAC;IACpC,eAAe,CAAC,EAAE,qBAAqB,EAAE,CAAC;CAC1C;AAED,MAAM,WAAW,kBAAkB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,qBAAqB;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;CACb;AAED,MAAM,MAAM,oBAAoB,GAC7B;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,oBAAoB,EAAE,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,GAC7F;IAAE,MAAM,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,oBAAoB,EAAE,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,CAAC;AAEnH,MAAM,WAAW,uBAAuB;IACvC,EAAE,EAAE,MAAM,CAAC;IACX,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,IAAI,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAAC;IACxC,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7C,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACtC;AAED,wBAAgB,oBAAoB,CAAC,IAAI,EAAE;IAC1C,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,SAAS,oBAAoB,EAAE,CAAC;IAC3C,OAAO,EAAE,kBAAkB,CAAC;CAC5B,GAAG,oBAAoB,CAkDvB","sourcesContent":["export type ResidencyControl = \"full\" | \"keep-alive\" | \"advisory\";\nexport type RuntimeRole = \"active\" | \"router\" | \"bench\" | \"probe\";\n\nexport interface RuntimeResidentModel {\n\tadapterId: string;\n\tmodel: string;\n\tbytes: number;\n\tlastUsedAtMs: number;\n\tpriority: number;\n\tpinned?: boolean;\n\tresidencyControl: ResidencyControl;\n}\n\nexport interface RuntimeLoadRequest {\n\tmodel: string;\n\tbytes: number;\n\trole: RuntimeRole;\n\tpriority: number;\n\tnowMs: number;\n\tminDwellMs?: number;\n\tpinActiveModel?: string;\n\treservations?: RuntimeReservation[];\n\trecentEvictions?: RuntimeEvictionRecord[];\n}\n\nexport interface RuntimeReservation {\n\tmodel: string;\n\tbytes: number;\n\tpriority: number;\n}\n\nexport interface RuntimeEvictionRecord {\n\tevicted: string;\n\tloaded: string;\n\tatMs: number;\n}\n\nexport type RuntimeResidencyPlan =\n\t| { status: \"fits\"; evict: RuntimeResidentModel[]; requiredBytes: number; budgetBytes: number }\n\t| { status: \"refuse\"; reason: string; evict: RuntimeResidentModel[]; requiredBytes: number; budgetBytes: number };\n\nexport interface RuntimeResidencyAdapter {\n\tid: string;\n\tresidencyControl: ResidencyControl;\n\tlist(): Promise<RuntimeResidentModel[]>;\n\tensureResident(model: string): Promise<void>;\n\trelease(model: string): Promise<void>;\n}\n\nexport function planRuntimeResidency(args: {\n\tbudgetBytes: number;\n\tresidents: readonly RuntimeResidentModel[];\n\trequest: RuntimeLoadRequest;\n}): RuntimeResidencyPlan {\n\tconst requestedModels = new Map<string, { bytes: number; priority: number }>();\n\trequestedModels.set(args.request.model, { bytes: args.request.bytes, priority: args.request.priority });\n\tfor (const reservation of args.request.reservations ?? []) {\n\t\tconst existing = requestedModels.get(reservation.model);\n\t\tif (!existing || reservation.bytes > existing.bytes) {\n\t\t\trequestedModels.set(reservation.model, { bytes: reservation.bytes, priority: reservation.priority });\n\t\t}\n\t}\n\n\tconst residentByModel = new Map(args.residents.map((resident) => [resident.model, resident]));\n\tconst residentBytes = args.residents.reduce((sum, resident) => sum + resident.bytes, 0);\n\tconst missingBytes = [...requestedModels].reduce((sum, [model, request]) => {\n\t\treturn residentByModel.has(model) ? sum : sum + request.bytes;\n\t}, 0);\n\tconst requiredBytes = residentBytes + missingBytes;\n\tif (requiredBytes <= args.budgetBytes) {\n\t\treturn { status: \"fits\", evict: [], requiredBytes, budgetBytes: args.budgetBytes };\n\t}\n\n\tif (hasPingPongRisk(args.request)) {\n\t\treturn { status: \"refuse\", reason: \"anti-thrash\", evict: [], requiredBytes, budgetBytes: args.budgetBytes };\n\t}\n\n\tlet projectedBytes = requiredBytes;\n\tconst evict: RuntimeResidentModel[] = [];\n\tconst candidates = args.residents\n\t\t.filter((resident) => !requestedModels.has(resident.model))\n\t\t.filter((resident) => !resident.pinned && resident.model !== args.request.pinActiveModel)\n\t\t.filter((resident) => resident.residencyControl !== \"advisory\")\n\t\t.filter(\n\t\t\t(resident) =>\n\t\t\t\targs.request.minDwellMs === undefined ||\n\t\t\t\targs.request.nowMs - resident.lastUsedAtMs >= args.request.minDwellMs,\n\t\t)\n\t\t.sort((left, right) => left.priority - right.priority || left.lastUsedAtMs - right.lastUsedAtMs);\n\tfor (const candidate of candidates) {\n\t\tevict.push(candidate);\n\t\tprojectedBytes -= candidate.bytes;\n\t\tif (projectedBytes <= args.budgetBytes) {\n\t\t\treturn { status: \"fits\", evict, requiredBytes, budgetBytes: args.budgetBytes };\n\t\t}\n\t}\n\treturn {\n\t\tstatus: \"refuse\",\n\t\treason: \"insufficient-evictable-memory\",\n\t\tevict,\n\t\trequiredBytes,\n\t\tbudgetBytes: args.budgetBytes,\n\t};\n}\n\nfunction hasPingPongRisk(request: RuntimeLoadRequest): boolean {\n\treturn (request.recentEvictions ?? []).some(\n\t\t(record) => record.evicted === request.model && request.nowMs - record.atMs <= (request.minDwellMs ?? 0),\n\t);\n}\n"]}
+{"version":3,"file":"runtime-arbiter.d.ts","sourceRoot":"","sources":["../../../src/core/models/runtime-arbiter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE7E,MAAM,MAAM,gBAAgB,GAAG,MAAM,GAAG,YAAY,GAAG,UAAU,CAAC;AAClE,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,CAAC;AAElE,MAAM,WAAW,oBAAoB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,gBAAgB,EAAE,gBAAgB,CAAC;CACnC;AAED,MAAM,WAAW,kBAAkB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,kBAAkB,EAAE,CAAC;IACpC,eAAe,CAAC,EAAE,qBAAqB,EAAE,CAAC;CAC1C;AAED,MAAM,WAAW,kBAAkB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,qBAAqB;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;CACb;AAED,MAAM,MAAM,oBAAoB,GAC7B;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,oBAAoB,EAAE,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,GAC7F;IAAE,MAAM,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,oBAAoB,EAAE,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,CAAC;AAEnH,MAAM,WAAW,uBAAuB;IACvC,EAAE,EAAE,MAAM,CAAC;IACX,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,IAAI,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAAC;IACxC,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7C,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACtC;AAED,qBAAa,uBAAuB;IACnC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAuC;IAEhE,YAAY,IAAI,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,uBAAuB,EAAE,CAAA;KAAE,EAG7E;IAEK,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAmBlG;CACD;AAED,qBAAa,6BAA8B,YAAW,uBAAuB;IAC5E,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,gBAAgB,eAAyB;IAClD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmF;IAC3G,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAe;IAErC,YACC,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,IAAI,CAAC,aAAa,EAAE,oBAAoB,GAAG,gBAAgB,GAAG,iBAAiB,CAAC,EACzF,OAAO,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,MAAM,CAAA;KAAO,EAKtC;IAEK,IAAI,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAU5C;IAEK,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGjD;IAEK,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE1C;CACD;AAED,qBAAa,mCAAoC,YAAW,uBAAuB;IAClF,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,gBAAgB,SAAmB;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAyD;IACjF,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAe;IAErC,YACC,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,IAAI,CAAC,mBAAmB,EAAE,QAAQ,GAAG,OAAO,GAAG,MAAM,CAAC,EAC/D,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,OAAO,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,MAAM,CAAA;KAAO,EAOtC;IAEK,IAAI,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAa5C;IAEK,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGlD;IAEK,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE3C;CACD;AAED,qBAAa,+BAAgC,YAAW,uBAAuB;IAC9E,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,gBAAgB,aAAuB;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAyB;IAEnD,YAAY,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,oBAAoB,EAAE,WAAW,GAAG,kBAAkB,CAAC,EAAE,EAOhG;IAEK,IAAI,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAE5C;IAEK,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAG;IAEhD,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAG;CAC/C;AAED,wBAAgB,oBAAoB,CAAC,IAAI,EAAE;IAC1C,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,SAAS,oBAAoB,EAAE,CAAC;IAC3C,OAAO,EAAE,kBAAkB,CAAC;CAC5B,GAAG,oBAAoB,CAkDvB","sourcesContent":["import type { OllamaRuntime, TransformersRuntime } from \"./local-runtime.ts\";\n\nexport type ResidencyControl = \"full\" | \"keep-alive\" | \"advisory\";\nexport type RuntimeRole = \"active\" | \"router\" | \"bench\" | \"probe\";\n\nexport interface RuntimeResidentModel {\n\tadapterId: string;\n\tmodel: string;\n\tbytes: number;\n\tlastUsedAtMs: number;\n\tpriority: number;\n\tpinned?: boolean;\n\tresidencyControl: ResidencyControl;\n}\n\nexport interface RuntimeLoadRequest {\n\tmodel: string;\n\tbytes: number;\n\trole: RuntimeRole;\n\tpriority: number;\n\tnowMs: number;\n\tminDwellMs?: number;\n\tpinActiveModel?: string;\n\treservations?: RuntimeReservation[];\n\trecentEvictions?: RuntimeEvictionRecord[];\n}\n\nexport interface RuntimeReservation {\n\tmodel: string;\n\tbytes: number;\n\tpriority: number;\n}\n\nexport interface RuntimeEvictionRecord {\n\tevicted: string;\n\tloaded: string;\n\tatMs: number;\n}\n\nexport type RuntimeResidencyPlan =\n\t| { status: \"fits\"; evict: RuntimeResidentModel[]; requiredBytes: number; budgetBytes: number }\n\t| { status: \"refuse\"; reason: string; evict: RuntimeResidentModel[]; requiredBytes: number; budgetBytes: number };\n\nexport interface RuntimeResidencyAdapter {\n\tid: string;\n\tresidencyControl: ResidencyControl;\n\tlist(): Promise<RuntimeResidentModel[]>;\n\tensureResident(model: string): Promise<void>;\n\trelease(model: string): Promise<void>;\n}\n\nexport class RuntimeResidencyArbiter {\n\tprivate readonly budgetBytes: number;\n\tprivate readonly adapters: Map<string, RuntimeResidencyAdapter>;\n\n\tconstructor(args: { budgetBytes: number; adapters: RuntimeResidencyAdapter[] }) {\n\t\tthis.budgetBytes = args.budgetBytes;\n\t\tthis.adapters = new Map(args.adapters.map((adapter) => [adapter.id, adapter]));\n\t}\n\n\tasync ensureResident(adapterId: string, request: RuntimeLoadRequest): Promise<RuntimeResidencyPlan> {\n\t\tconst adapter = this.adapters.get(adapterId);\n\t\tconst residents = (await Promise.all([...this.adapters.values()].map((entry) => entry.list()))).flat();\n\t\tif (!adapter) {\n\t\t\treturn {\n\t\t\t\tstatus: \"refuse\",\n\t\t\t\treason: \"unknown-adapter\",\n\t\t\t\tevict: [],\n\t\t\t\trequiredBytes: request.bytes,\n\t\t\t\tbudgetBytes: this.budgetBytes,\n\t\t\t};\n\t\t}\n\t\tconst plan = planRuntimeResidency({ budgetBytes: this.budgetBytes, residents, request });\n\t\tif (plan.status !== \"fits\") return plan;\n\t\tfor (const resident of plan.evict) {\n\t\t\tawait this.adapters.get(resident.adapterId)?.release(resident.model);\n\t\t}\n\t\tawait adapter.ensureResident(request.model);\n\t\treturn plan;\n\t}\n}\n\nexport class OllamaRuntimeResidencyAdapter implements RuntimeResidencyAdapter {\n\treadonly id: string;\n\treadonly residencyControl = \"keep-alive\" as const;\n\tprivate readonly runtime: Pick<OllamaRuntime, \"listResidentModels\" | \"ensureResident\" | \"releaseResident\">;\n\tprivate readonly nowMs: () => number;\n\n\tconstructor(\n\t\tid: string,\n\t\truntime: Pick<OllamaRuntime, \"listResidentModels\" | \"ensureResident\" | \"releaseResident\">,\n\t\toptions: { nowMs?: () => number } = {},\n\t) {\n\t\tthis.id = id;\n\t\tthis.runtime = runtime;\n\t\tthis.nowMs = options.nowMs ?? Date.now;\n\t}\n\n\tasync list(): Promise<RuntimeResidentModel[]> {\n\t\tconst now = this.nowMs();\n\t\treturn (await this.runtime.listResidentModels()).map((model) => ({\n\t\t\tadapterId: this.id,\n\t\t\tmodel: model.name,\n\t\t\tbytes: model.sizeBytes,\n\t\t\tlastUsedAtMs: now,\n\t\t\tpriority: 10,\n\t\t\tresidencyControl: this.residencyControl,\n\t\t}));\n\t}\n\n\tasync ensureResident(model: string): Promise<void> {\n\t\tconst result = await this.runtime.ensureResident(model);\n\t\tif (!result.ok) throw new Error(result.error);\n\t}\n\n\tasync release(model: string): Promise<void> {\n\t\tawait this.runtime.releaseResident(model);\n\t}\n}\n\nexport class TransformersRuntimeResidencyAdapter implements RuntimeResidencyAdapter {\n\treadonly id: string;\n\treadonly residencyControl = \"full\" as const;\n\tprivate readonly runtime: Pick<TransformersRuntime, \"detect\" | \"start\" | \"stop\">;\n\tprivate readonly model: string;\n\tprivate readonly bytes: number;\n\tprivate readonly nowMs: () => number;\n\n\tconstructor(\n\t\tid: string,\n\t\truntime: Pick<TransformersRuntime, \"detect\" | \"start\" | \"stop\">,\n\t\tmodel: string,\n\t\tbytes: number,\n\t\toptions: { nowMs?: () => number } = {},\n\t) {\n\t\tthis.id = id;\n\t\tthis.runtime = runtime;\n\t\tthis.model = model;\n\t\tthis.bytes = bytes;\n\t\tthis.nowMs = options.nowMs ?? Date.now;\n\t}\n\n\tasync list(): Promise<RuntimeResidentModel[]> {\n\t\tconst status = await this.runtime.detect();\n\t\tif (!status.serverUp) return [];\n\t\treturn [\n\t\t\t{\n\t\t\t\tadapterId: this.id,\n\t\t\t\tmodel: this.model,\n\t\t\t\tbytes: this.bytes,\n\t\t\t\tlastUsedAtMs: this.nowMs(),\n\t\t\t\tpriority: 10,\n\t\t\t\tresidencyControl: this.residencyControl,\n\t\t\t},\n\t\t];\n\t}\n\n\tasync ensureResident(_model: string): Promise<void> {\n\t\tconst result = await this.runtime.start();\n\t\tif (!result.started && result.reason !== \"already_running\") throw new Error(result.reason);\n\t}\n\n\tasync release(_model: string): Promise<void> {\n\t\tthis.runtime.stop();\n\t}\n}\n\nexport class AdvisoryRuntimeResidencyAdapter implements RuntimeResidencyAdapter {\n\treadonly id: string;\n\treadonly residencyControl = \"advisory\" as const;\n\tprivate readonly residents: RuntimeResidentModel[];\n\n\tconstructor(id: string, residents: Omit<RuntimeResidentModel, \"adapterId\" | \"residencyControl\">[]) {\n\t\tthis.id = id;\n\t\tthis.residents = residents.map((resident) => ({\n\t\t\t...resident,\n\t\t\tadapterId: id,\n\t\t\tresidencyControl: this.residencyControl,\n\t\t}));\n\t}\n\n\tasync list(): Promise<RuntimeResidentModel[]> {\n\t\treturn [...this.residents];\n\t}\n\n\tasync ensureResident(_model: string): Promise<void> {}\n\n\tasync release(_model: string): Promise<void> {}\n}\n\nexport function planRuntimeResidency(args: {\n\tbudgetBytes: number;\n\tresidents: readonly RuntimeResidentModel[];\n\trequest: RuntimeLoadRequest;\n}): RuntimeResidencyPlan {\n\tconst requestedModels = new Map<string, { bytes: number; priority: number }>();\n\trequestedModels.set(args.request.model, { bytes: args.request.bytes, priority: args.request.priority });\n\tfor (const reservation of args.request.reservations ?? []) {\n\t\tconst existing = requestedModels.get(reservation.model);\n\t\tif (!existing || reservation.bytes > existing.bytes) {\n\t\t\trequestedModels.set(reservation.model, { bytes: reservation.bytes, priority: reservation.priority });\n\t\t}\n\t}\n\n\tconst residentByModel = new Map(args.residents.map((resident) => [resident.model, resident]));\n\tconst residentBytes = args.residents.reduce((sum, resident) => sum + resident.bytes, 0);\n\tconst missingBytes = [...requestedModels].reduce((sum, [model, request]) => {\n\t\treturn residentByModel.has(model) ? sum : sum + request.bytes;\n\t}, 0);\n\tconst requiredBytes = residentBytes + missingBytes;\n\tif (requiredBytes <= args.budgetBytes) {\n\t\treturn { status: \"fits\", evict: [], requiredBytes, budgetBytes: args.budgetBytes };\n\t}\n\n\tif (hasPingPongRisk(args.request)) {\n\t\treturn { status: \"refuse\", reason: \"anti-thrash\", evict: [], requiredBytes, budgetBytes: args.budgetBytes };\n\t}\n\n\tlet projectedBytes = requiredBytes;\n\tconst evict: RuntimeResidentModel[] = [];\n\tconst candidates = args.residents\n\t\t.filter((resident) => !requestedModels.has(resident.model))\n\t\t.filter((resident) => !resident.pinned && resident.model !== args.request.pinActiveModel)\n\t\t.filter((resident) => resident.residencyControl !== \"advisory\")\n\t\t.filter(\n\t\t\t(resident) =>\n\t\t\t\targs.request.minDwellMs === undefined ||\n\t\t\t\targs.request.nowMs - resident.lastUsedAtMs >= args.request.minDwellMs,\n\t\t)\n\t\t.sort((left, right) => left.priority - right.priority || left.lastUsedAtMs - right.lastUsedAtMs);\n\tfor (const candidate of candidates) {\n\t\tevict.push(candidate);\n\t\tprojectedBytes -= candidate.bytes;\n\t\tif (projectedBytes <= args.budgetBytes) {\n\t\t\treturn { status: \"fits\", evict, requiredBytes, budgetBytes: args.budgetBytes };\n\t\t}\n\t}\n\treturn {\n\t\tstatus: \"refuse\",\n\t\treason: \"insufficient-evictable-memory\",\n\t\tevict,\n\t\trequiredBytes,\n\t\tbudgetBytes: args.budgetBytes,\n\t};\n}\n\nfunction hasPingPongRisk(request: RuntimeLoadRequest): boolean {\n\treturn (request.recentEvictions ?? []).some(\n\t\t(record) => record.evicted === request.model && request.nowMs - record.atMs <= (request.minDwellMs ?? 0),\n\t);\n}\n"]}

package/dist/core/models/runtime-arbiter.js

@@ -1,3 +1,118 @@
+export class RuntimeResidencyArbiter {
+    budgetBytes;
+    adapters;
+    constructor(args) {
+        this.budgetBytes = args.budgetBytes;
+        this.adapters = new Map(args.adapters.map((adapter) => [adapter.id, adapter]));
+    }
+    async ensureResident(adapterId, request) {
+        const adapter = this.adapters.get(adapterId);
+        const residents = (await Promise.all([...this.adapters.values()].map((entry) => entry.list()))).flat();
+        if (!adapter) {
+            return {
+                status: "refuse",
+                reason: "unknown-adapter",
+                evict: [],
+                requiredBytes: request.bytes,
+                budgetBytes: this.budgetBytes,
+            };
+        }
+        const plan = planRuntimeResidency({ budgetBytes: this.budgetBytes, residents, request });
+        if (plan.status !== "fits")
+            return plan;
+        for (const resident of plan.evict) {
+            await this.adapters.get(resident.adapterId)?.release(resident.model);
+        }
+        await adapter.ensureResident(request.model);
+        return plan;
+    }
+}
+export class OllamaRuntimeResidencyAdapter {
+    id;
+    residencyControl = "keep-alive";
+    runtime;
+    nowMs;
+    constructor(id, runtime, options = {}) {
+        this.id = id;
+        this.runtime = runtime;
+        this.nowMs = options.nowMs ?? Date.now;
+    }
+    async list() {
+        const now = this.nowMs();
+        return (await this.runtime.listResidentModels()).map((model) => ({
+            adapterId: this.id,
+            model: model.name,
+            bytes: model.sizeBytes,
+            lastUsedAtMs: now,
+            priority: 10,
+            residencyControl: this.residencyControl,
+        }));
+    }
+    async ensureResident(model) {
+        const result = await this.runtime.ensureResident(model);
+        if (!result.ok)
+            throw new Error(result.error);
+    }
+    async release(model) {
+        await this.runtime.releaseResident(model);
+    }
+}
+export class TransformersRuntimeResidencyAdapter {
+    id;
+    residencyControl = "full";
+    runtime;
+    model;
+    bytes;
+    nowMs;
+    constructor(id, runtime, model, bytes, options = {}) {
+        this.id = id;
+        this.runtime = runtime;
+        this.model = model;
+        this.bytes = bytes;
+        this.nowMs = options.nowMs ?? Date.now;
+    }
+    async list() {
+        const status = await this.runtime.detect();
+        if (!status.serverUp)
+            return [];
+        return [
+            {
+                adapterId: this.id,
+                model: this.model,
+                bytes: this.bytes,
+                lastUsedAtMs: this.nowMs(),
+                priority: 10,
+                residencyControl: this.residencyControl,
+            },
+        ];
+    }
+    async ensureResident(_model) {
+        const result = await this.runtime.start();
+        if (!result.started && result.reason !== "already_running")
+            throw new Error(result.reason);
+    }
+    async release(_model) {
+        this.runtime.stop();
+    }
+}
+export class AdvisoryRuntimeResidencyAdapter {
+    id;
+    residencyControl = "advisory";
+    residents;
+    constructor(id, residents) {
+        this.id = id;
+        this.residents = residents.map((resident) => ({
+            ...resident,
+            adapterId: id,
+            residencyControl: this.residencyControl,
+        }));
+    }
+    async list() {
+        return [...this.residents];
+    }
+    async ensureResident(_model) { }
+    async release(_model) { }
+}
 export function planRuntimeResidency(args) {
     const requestedModels = new Map();
     requestedModels.set(args.request.model, { bytes: args.request.bytes, priority: args.request.priority });

package/dist/core/models/runtime-arbiter.js.map

@@ -1 +1 @@
-{"version":3,"file":"runtime-arbiter.js","sourceRoot":"","sources":["../../../src/core/models/runtime-arbiter.ts"],"names":[],"mappings":"AAiDA,MAAM,UAAU,oBAAoB,CAAC,IAIpC,EAAwB;IACxB,MAAM,eAAe,GAAG,IAAI,GAAG,EAA+C,CAAC;IAC/E,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IACxG,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,EAAE,CAAC;QAC3D,MAAM,QAAQ,GAAG,eAAe,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACxD,IAAI,CAAC,QAAQ,IAAI,WAAW,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;YACrD,eAAe,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,QAAQ,EAAE,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAC;QACtG,CAAC;IACF,CAAC;IAED,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC9F,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IACxF,MAAM,YAAY,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC;QAC3E,OAAO,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC;IAAA,CAC9D,EAAE,CAAC,CAAC,CAAC;IACN,MAAM,aAAa,GAAG,aAAa,GAAG,YAAY,CAAC;IACnD,IAAI,aAAa,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;QACvC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,aAAa,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC;IACpF,CAAC;IAED,IAAI,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACnC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,EAAE,EAAE,aAAa,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC;IAC7G,CAAC;IAED,IAAI,cAAc,GAAG,aAAa,CAAC;IACnC,MAAM,KAAK,GAA2B,EAAE,CAAC;IACzC,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS;SAC/B,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;SAC1D,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,KAAK,KAAK,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC;SACxF,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,gBAAgB,KAAK,UAAU,CAAC;SAC9D,MAAM,CACN,CAAC,QAAQ,EAAE,EAAE,CACZ,IAAI,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS;QACrC,IAAI,CAAC,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC,YAAY,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CACtE;SACA,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC;IAClG,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,cAAc,IAAI,SAAS,CAAC,KAAK,CAAC;QAClC,IAAI,cAAc,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACxC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC;QAChF,CAAC;IACF,CAAC;IACD,OAAO;QACN,MAAM,EAAE,QAAQ;QAChB,MAAM,EAAE,+BAA+B;QACvC,KAAK;QACL,aAAa;QACb,WAAW,EAAE,IAAI,CAAC,WAAW;KAC7B,CAAC;AAAA,CACF;AAED,SAAS,eAAe,CAAC,OAA2B,EAAW;IAC9D,OAAO,CAAC,OAAO,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,IAAI,CAC1C,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,KAAK,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,CAAC,CAAC,CACxG,CAAC;AAAA,CACF","sourcesContent":["export type ResidencyControl = \"full\" | \"keep-alive\" | \"advisory\";\nexport type RuntimeRole = \"active\" | \"router\" | \"bench\" | \"probe\";\n\nexport interface RuntimeResidentModel {\n\tadapterId: string;\n\tmodel: string;\n\tbytes: number;\n\tlastUsedAtMs: number;\n\tpriority: number;\n\tpinned?: boolean;\n\tresidencyControl: ResidencyControl;\n}\n\nexport interface RuntimeLoadRequest {\n\tmodel: string;\n\tbytes: number;\n\trole: RuntimeRole;\n\tpriority: number;\n\tnowMs: number;\n\tminDwellMs?: number;\n\tpinActiveModel?: string;\n\treservations?: RuntimeReservation[];\n\trecentEvictions?: RuntimeEvictionRecord[];\n}\n\nexport interface RuntimeReservation {\n\tmodel: string;\n\tbytes: number;\n\tpriority: number;\n}\n\nexport interface RuntimeEvictionRecord {\n\tevicted: string;\n\tloaded: string;\n\tatMs: number;\n}\n\nexport type RuntimeResidencyPlan =\n\t| { status: \"fits\"; evict: RuntimeResidentModel[]; requiredBytes: number; budgetBytes: number }\n\t| { status: \"refuse\"; reason: string; evict: RuntimeResidentModel[]; requiredBytes: number; budgetBytes: number };\n\nexport interface RuntimeResidencyAdapter {\n\tid: string;\n\tresidencyControl: ResidencyControl;\n\tlist(): Promise<RuntimeResidentModel[]>;\n\tensureResident(model: string): Promise<void>;\n\trelease(model: string): Promise<void>;\n}\n\nexport function planRuntimeResidency(args: {\n\tbudgetBytes: number;\n\tresidents: readonly RuntimeResidentModel[];\n\trequest: RuntimeLoadRequest;\n}): RuntimeResidencyPlan {\n\tconst requestedModels = new Map<string, { bytes: number; priority: number }>();\n\trequestedModels.set(args.request.model, { bytes: args.request.bytes, priority: args.request.priority });\n\tfor (const reservation of args.request.reservations ?? []) {\n\t\tconst existing = requestedModels.get(reservation.model);\n\t\tif (!existing || reservation.bytes > existing.bytes) {\n\t\t\trequestedModels.set(reservation.model, { bytes: reservation.bytes, priority: reservation.priority });\n\t\t}\n\t}\n\n\tconst residentByModel = new Map(args.residents.map((resident) => [resident.model, resident]));\n\tconst residentBytes = args.residents.reduce((sum, resident) => sum + resident.bytes, 0);\n\tconst missingBytes = [...requestedModels].reduce((sum, [model, request]) => {\n\t\treturn residentByModel.has(model) ? sum : sum + request.bytes;\n\t}, 0);\n\tconst requiredBytes = residentBytes + missingBytes;\n\tif (requiredBytes <= args.budgetBytes) {\n\t\treturn { status: \"fits\", evict: [], requiredBytes, budgetBytes: args.budgetBytes };\n\t}\n\n\tif (hasPingPongRisk(args.request)) {\n\t\treturn { status: \"refuse\", reason: \"anti-thrash\", evict: [], requiredBytes, budgetBytes: args.budgetBytes };\n\t}\n\n\tlet projectedBytes = requiredBytes;\n\tconst evict: RuntimeResidentModel[] = [];\n\tconst candidates = args.residents\n\t\t.filter((resident) => !requestedModels.has(resident.model))\n\t\t.filter((resident) => !resident.pinned && resident.model !== args.request.pinActiveModel)\n\t\t.filter((resident) => resident.residencyControl !== \"advisory\")\n\t\t.filter(\n\t\t\t(resident) =>\n\t\t\t\targs.request.minDwellMs === undefined ||\n\t\t\t\targs.request.nowMs - resident.lastUsedAtMs >= args.request.minDwellMs,\n\t\t)\n\t\t.sort((left, right) => left.priority - right.priority || left.lastUsedAtMs - right.lastUsedAtMs);\n\tfor (const candidate of candidates) {\n\t\tevict.push(candidate);\n\t\tprojectedBytes -= candidate.bytes;\n\t\tif (projectedBytes <= args.budgetBytes) {\n\t\t\treturn { status: \"fits\", evict, requiredBytes, budgetBytes: args.budgetBytes };\n\t\t}\n\t}\n\treturn {\n\t\tstatus: \"refuse\",\n\t\treason: \"insufficient-evictable-memory\",\n\t\tevict,\n\t\trequiredBytes,\n\t\tbudgetBytes: args.budgetBytes,\n\t};\n}\n\nfunction hasPingPongRisk(request: RuntimeLoadRequest): boolean {\n\treturn (request.recentEvictions ?? []).some(\n\t\t(record) => record.evicted === request.model && request.nowMs - record.atMs <= (request.minDwellMs ?? 0),\n\t);\n}\n"]}
+{"version":3,"file":"runtime-arbiter.js","sourceRoot":"","sources":["../../../src/core/models/runtime-arbiter.ts"],"names":[],"mappings":"AAmDA,MAAM,OAAO,uBAAuB;IAClB,WAAW,CAAS;IACpB,QAAQ,CAAuC;IAEhE,YAAY,IAAkE,EAAE;QAC/E,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;QACpC,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAAA,CAC/E;IAED,KAAK,CAAC,cAAc,CAAC,SAAiB,EAAE,OAA2B,EAAiC;QACnG,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC7C,MAAM,SAAS,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACvG,IAAI,CAAC,OAAO,EAAE,CAAC;YACd,OAAO;gBACN,MAAM,EAAE,QAAQ;gBAChB,MAAM,EAAE,iBAAiB;gBACzB,KAAK,EAAE,EAAE;gBACT,aAAa,EAAE,OAAO,CAAC,KAAK;gBAC5B,WAAW,EAAE,IAAI,CAAC,WAAW;aAC7B,CAAC;QACH,CAAC;QACD,MAAM,IAAI,GAAG,oBAAoB,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;QACzF,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM;YAAE,OAAO,IAAI,CAAC;QACxC,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACnC,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACtE,CAAC;QACD,MAAM,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC5C,OAAO,IAAI,CAAC;IAAA,CACZ;CACD;AAED,MAAM,OAAO,6BAA6B;IAChC,EAAE,CAAS;IACX,gBAAgB,GAAG,YAAqB,CAAC;IACjC,OAAO,CAAmF;IAC1F,KAAK,CAAe;IAErC,YACC,EAAU,EACV,OAAyF,EACzF,OAAO,GAA6B,EAAE,EACrC;QACD,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC;IAAA,CACvC;IAED,KAAK,CAAC,IAAI,GAAoC;QAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QACzB,OAAO,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAChE,SAAS,EAAE,IAAI,CAAC,EAAE;YAClB,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,KAAK,EAAE,KAAK,CAAC,SAAS;YACtB,YAAY,EAAE,GAAG;YACjB,QAAQ,EAAE,EAAE;YACZ,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;SACvC,CAAC,CAAC,CAAC;IAAA,CACJ;IAED,KAAK,CAAC,cAAc,CAAC,KAAa,EAAiB;QAClD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QACxD,IAAI,CAAC,MAAM,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAAA,CAC9C;IAED,KAAK,CAAC,OAAO,CAAC,KAAa,EAAiB;QAC3C,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;IAAA,CAC1C;CACD;AAED,MAAM,OAAO,mCAAmC;IACtC,EAAE,CAAS;IACX,gBAAgB,GAAG,MAAe,CAAC;IAC3B,OAAO,CAAyD;IAChE,KAAK,CAAS;IACd,KAAK,CAAS;IACd,KAAK,CAAe;IAErC,YACC,EAAU,EACV,OAA+D,EAC/D,KAAa,EACb,KAAa,EACb,OAAO,GAA6B,EAAE,EACrC;QACD,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC;IAAA,CACvC;IAED,KAAK,CAAC,IAAI,GAAoC;QAC7C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAC3C,IAAI,CAAC,MAAM,CAAC,QAAQ;YAAE,OAAO,EAAE,CAAC;QAChC,OAAO;YACN;gBACC,SAAS,EAAE,IAAI,CAAC,EAAE;gBAClB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,YAAY,EAAE,IAAI,CAAC,KAAK,EAAE;gBAC1B,QAAQ,EAAE,EAAE;gBACZ,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;aACvC;SACD,CAAC;IAAA,CACF;IAED,KAAK,CAAC,cAAc,CAAC,MAAc,EAAiB;QACnD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAC1C,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,MAAM,KAAK,iBAAiB;YAAE,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAAA,CAC3F;IAED,KAAK,CAAC,OAAO,CAAC,MAAc,EAAiB;QAC5C,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IAAA,CACpB;CACD;AAED,MAAM,OAAO,+BAA+B;IAClC,EAAE,CAAS;IACX,gBAAgB,GAAG,UAAmB,CAAC;IAC/B,SAAS,CAAyB;IAEnD,YAAY,EAAU,EAAE,SAAyE,EAAE;QAClG,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAC7C,GAAG,QAAQ;YACX,SAAS,EAAE,EAAE;YACb,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;SACvC,CAAC,CAAC,CAAC;IAAA,CACJ;IAED,KAAK,CAAC,IAAI,GAAoC;QAC7C,OAAO,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;IAAA,CAC3B;IAED,KAAK,CAAC,cAAc,CAAC,MAAc,EAAiB,EAAC,CAAC;IAEtD,KAAK,CAAC,OAAO,CAAC,MAAc,EAAiB,EAAC,CAAC;CAC/C;AAED,MAAM,UAAU,oBAAoB,CAAC,IAIpC,EAAwB;IACxB,MAAM,eAAe,GAAG,IAAI,GAAG,EAA+C,CAAC;IAC/E,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IACxG,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,EAAE,CAAC;QAC3D,MAAM,QAAQ,GAAG,eAAe,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACxD,IAAI,CAAC,QAAQ,IAAI,WAAW,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;YACrD,eAAe,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,QAAQ,EAAE,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAC;QACtG,CAAC;IACF,CAAC;IAED,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC9F,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IACxF,MAAM,YAAY,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC;QAC3E,OAAO,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC;IAAA,CAC9D,EAAE,CAAC,CAAC,CAAC;IACN,MAAM,aAAa,GAAG,aAAa,GAAG,YAAY,CAAC;IACnD,IAAI,aAAa,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;QACvC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,aAAa,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC;IACpF,CAAC;IAED,IAAI,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACnC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,EAAE,EAAE,aAAa,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC;IAC7G,CAAC;IAED,IAAI,cAAc,GAAG,aAAa,CAAC;IACnC,MAAM,KAAK,GAA2B,EAAE,CAAC;IACzC,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS;SAC/B,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;SAC1D,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,KAAK,KAAK,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC;SACxF,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,gBAAgB,KAAK,UAAU,CAAC;SAC9D,MAAM,CACN,CAAC,QAAQ,EAAE,EAAE,CACZ,IAAI,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS;QACrC,IAAI,CAAC,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC,YAAY,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CACtE;SACA,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC;IAClG,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,cAAc,IAAI,SAAS,CAAC,KAAK,CAAC;QAClC,IAAI,cAAc,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACxC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC;QAChF,CAAC;IACF,CAAC;IACD,OAAO;QACN,MAAM,EAAE,QAAQ;QAChB,MAAM,EAAE,+BAA+B;QACvC,KAAK;QACL,aAAa;QACb,WAAW,EAAE,IAAI,CAAC,WAAW;KAC7B,CAAC;AAAA,CACF;AAED,SAAS,eAAe,CAAC,OAA2B,EAAW;IAC9D,OAAO,CAAC,OAAO,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,IAAI,CAC1C,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,KAAK,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,CAAC,CAAC,CACxG,CAAC;AAAA,CACF","sourcesContent":["import type { OllamaRuntime, TransformersRuntime } from \"./local-runtime.ts\";\n\nexport type ResidencyControl = \"full\" | \"keep-alive\" | \"advisory\";\nexport type RuntimeRole = \"active\" | \"router\" | \"bench\" | \"probe\";\n\nexport interface RuntimeResidentModel {\n\tadapterId: string;\n\tmodel: string;\n\tbytes: number;\n\tlastUsedAtMs: number;\n\tpriority: number;\n\tpinned?: boolean;\n\tresidencyControl: ResidencyControl;\n}\n\nexport interface RuntimeLoadRequest {\n\tmodel: string;\n\tbytes: number;\n\trole: RuntimeRole;\n\tpriority: number;\n\tnowMs: number;\n\tminDwellMs?: number;\n\tpinActiveModel?: string;\n\treservations?: RuntimeReservation[];\n\trecentEvictions?: RuntimeEvictionRecord[];\n}\n\nexport interface RuntimeReservation {\n\tmodel: string;\n\tbytes: number;\n\tpriority: number;\n}\n\nexport interface RuntimeEvictionRecord {\n\tevicted: string;\n\tloaded: string;\n\tatMs: number;\n}\n\nexport type RuntimeResidencyPlan =\n\t| { status: \"fits\"; evict: RuntimeResidentModel[]; requiredBytes: number; budgetBytes: number }\n\t| { status: \"refuse\"; reason: string; evict: RuntimeResidentModel[]; requiredBytes: number; budgetBytes: number };\n\nexport interface RuntimeResidencyAdapter {\n\tid: string;\n\tresidencyControl: ResidencyControl;\n\tlist(): Promise<RuntimeResidentModel[]>;\n\tensureResident(model: string): Promise<void>;\n\trelease(model: string): Promise<void>;\n}\n\nexport class RuntimeResidencyArbiter {\n\tprivate readonly budgetBytes: number;\n\tprivate readonly adapters: Map<string, RuntimeResidencyAdapter>;\n\n\tconstructor(args: { budgetBytes: number; adapters: RuntimeResidencyAdapter[] }) {\n\t\tthis.budgetBytes = args.budgetBytes;\n\t\tthis.adapters = new Map(args.adapters.map((adapter) => [adapter.id, adapter]));\n\t}\n\n\tasync ensureResident(adapterId: string, request: RuntimeLoadRequest): Promise<RuntimeResidencyPlan> {\n\t\tconst adapter = this.adapters.get(adapterId);\n\t\tconst residents = (await Promise.all([...this.adapters.values()].map((entry) => entry.list()))).flat();\n\t\tif (!adapter) {\n\t\t\treturn {\n\t\t\t\tstatus: \"refuse\",\n\t\t\t\treason: \"unknown-adapter\",\n\t\t\t\tevict: [],\n\t\t\t\trequiredBytes: request.bytes,\n\t\t\t\tbudgetBytes: this.budgetBytes,\n\t\t\t};\n\t\t}\n\t\tconst plan = planRuntimeResidency({ budgetBytes: this.budgetBytes, residents, request });\n\t\tif (plan.status !== \"fits\") return plan;\n\t\tfor (const resident of plan.evict) {\n\t\t\tawait this.adapters.get(resident.adapterId)?.release(resident.model);\n\t\t}\n\t\tawait adapter.ensureResident(request.model);\n\t\treturn plan;\n\t}\n}\n\nexport class OllamaRuntimeResidencyAdapter implements RuntimeResidencyAdapter {\n\treadonly id: string;\n\treadonly residencyControl = \"keep-alive\" as const;\n\tprivate readonly runtime: Pick<OllamaRuntime, \"listResidentModels\" | \"ensureResident\" | \"releaseResident\">;\n\tprivate readonly nowMs: () => number;\n\n\tconstructor(\n\t\tid: string,\n\t\truntime: Pick<OllamaRuntime, \"listResidentModels\" | \"ensureResident\" | \"releaseResident\">,\n\t\toptions: { nowMs?: () => number } = {},\n\t) {\n\t\tthis.id = id;\n\t\tthis.runtime = runtime;\n\t\tthis.nowMs = options.nowMs ?? Date.now;\n\t}\n\n\tasync list(): Promise<RuntimeResidentModel[]> {\n\t\tconst now = this.nowMs();\n\t\treturn (await this.runtime.listResidentModels()).map((model) => ({\n\t\t\tadapterId: this.id,\n\t\t\tmodel: model.name,\n\t\t\tbytes: model.sizeBytes,\n\t\t\tlastUsedAtMs: now,\n\t\t\tpriority: 10,\n\t\t\tresidencyControl: this.residencyControl,\n\t\t}));\n\t}\n\n\tasync ensureResident(model: string): Promise<void> {\n\t\tconst result = await this.runtime.ensureResident(model);\n\t\tif (!result.ok) throw new Error(result.error);\n\t}\n\n\tasync release(model: string): Promise<void> {\n\t\tawait this.runtime.releaseResident(model);\n\t}\n}\n\nexport class TransformersRuntimeResidencyAdapter implements RuntimeResidencyAdapter {\n\treadonly id: string;\n\treadonly residencyControl = \"full\" as const;\n\tprivate readonly runtime: Pick<TransformersRuntime, \"detect\" | \"start\" | \"stop\">;\n\tprivate readonly model: string;\n\tprivate readonly bytes: number;\n\tprivate readonly nowMs: () => number;\n\n\tconstructor(\n\t\tid: string,\n\t\truntime: Pick<TransformersRuntime, \"detect\" | \"start\" | \"stop\">,\n\t\tmodel: string,\n\t\tbytes: number,\n\t\toptions: { nowMs?: () => number } = {},\n\t) {\n\t\tthis.id = id;\n\t\tthis.runtime = runtime;\n\t\tthis.model = model;\n\t\tthis.bytes = bytes;\n\t\tthis.nowMs = options.nowMs ?? Date.now;\n\t}\n\n\tasync list(): Promise<RuntimeResidentModel[]> {\n\t\tconst status = await this.runtime.detect();\n\t\tif (!status.serverUp) return [];\n\t\treturn [\n\t\t\t{\n\t\t\t\tadapterId: this.id,\n\t\t\t\tmodel: this.model,\n\t\t\t\tbytes: this.bytes,\n\t\t\t\tlastUsedAtMs: this.nowMs(),\n\t\t\t\tpriority: 10,\n\t\t\t\tresidencyControl: this.residencyControl,\n\t\t\t},\n\t\t];\n\t}\n\n\tasync ensureResident(_model: string): Promise<void> {\n\t\tconst result = await this.runtime.start();\n\t\tif (!result.started && result.reason !== \"already_running\") throw new Error(result.reason);\n\t}\n\n\tasync release(_model: string): Promise<void> {\n\t\tthis.runtime.stop();\n\t}\n}\n\nexport class AdvisoryRuntimeResidencyAdapter implements RuntimeResidencyAdapter {\n\treadonly id: string;\n\treadonly residencyControl = \"advisory\" as const;\n\tprivate readonly residents: RuntimeResidentModel[];\n\n\tconstructor(id: string, residents: Omit<RuntimeResidentModel, \"adapterId\" | \"residencyControl\">[]) {\n\t\tthis.id = id;\n\t\tthis.residents = residents.map((resident) => ({\n\t\t\t...resident,\n\t\t\tadapterId: id,\n\t\t\tresidencyControl: this.residencyControl,\n\t\t}));\n\t}\n\n\tasync list(): Promise<RuntimeResidentModel[]> {\n\t\treturn [...this.residents];\n\t}\n\n\tasync ensureResident(_model: string): Promise<void> {}\n\n\tasync release(_model: string): Promise<void> {}\n}\n\nexport function planRuntimeResidency(args: {\n\tbudgetBytes: number;\n\tresidents: readonly RuntimeResidentModel[];\n\trequest: RuntimeLoadRequest;\n}): RuntimeResidencyPlan {\n\tconst requestedModels = new Map<string, { bytes: number; priority: number }>();\n\trequestedModels.set(args.request.model, { bytes: args.request.bytes, priority: args.request.priority });\n\tfor (const reservation of args.request.reservations ?? []) {\n\t\tconst existing = requestedModels.get(reservation.model);\n\t\tif (!existing || reservation.bytes > existing.bytes) {\n\t\t\trequestedModels.set(reservation.model, { bytes: reservation.bytes, priority: reservation.priority });\n\t\t}\n\t}\n\n\tconst residentByModel = new Map(args.residents.map((resident) => [resident.model, resident]));\n\tconst residentBytes = args.residents.reduce((sum, resident) => sum + resident.bytes, 0);\n\tconst missingBytes = [...requestedModels].reduce((sum, [model, request]) => {\n\t\treturn residentByModel.has(model) ? sum : sum + request.bytes;\n\t}, 0);\n\tconst requiredBytes = residentBytes + missingBytes;\n\tif (requiredBytes <= args.budgetBytes) {\n\t\treturn { status: \"fits\", evict: [], requiredBytes, budgetBytes: args.budgetBytes };\n\t}\n\n\tif (hasPingPongRisk(args.request)) {\n\t\treturn { status: \"refuse\", reason: \"anti-thrash\", evict: [], requiredBytes, budgetBytes: args.budgetBytes };\n\t}\n\n\tlet projectedBytes = requiredBytes;\n\tconst evict: RuntimeResidentModel[] = [];\n\tconst candidates = args.residents\n\t\t.filter((resident) => !requestedModels.has(resident.model))\n\t\t.filter((resident) => !resident.pinned && resident.model !== args.request.pinActiveModel)\n\t\t.filter((resident) => resident.residencyControl !== \"advisory\")\n\t\t.filter(\n\t\t\t(resident) =>\n\t\t\t\targs.request.minDwellMs === undefined ||\n\t\t\t\targs.request.nowMs - resident.lastUsedAtMs >= args.request.minDwellMs,\n\t\t)\n\t\t.sort((left, right) => left.priority - right.priority || left.lastUsedAtMs - right.lastUsedAtMs);\n\tfor (const candidate of candidates) {\n\t\tevict.push(candidate);\n\t\tprojectedBytes -= candidate.bytes;\n\t\tif (projectedBytes <= args.budgetBytes) {\n\t\t\treturn { status: \"fits\", evict, requiredBytes, budgetBytes: args.budgetBytes };\n\t\t}\n\t}\n\treturn {\n\t\tstatus: \"refuse\",\n\t\treason: \"insufficient-evictable-memory\",\n\t\tevict,\n\t\trequiredBytes,\n\t\tbudgetBytes: args.budgetBytes,\n\t};\n}\n\nfunction hasPingPongRisk(request: RuntimeLoadRequest): boolean {\n\treturn (request.recentEvictions ?? []).some(\n\t\t(record) => record.evicted === request.model && request.nowMs - record.atMs <= (request.minDwellMs ?? 0),\n\t);\n}\n"]}

package/dist/modes/interactive/interactive-mode.d.ts

@@ -186,6 +186,9 @@ export declare class InteractiveMode {
     private handleEvent;
     /** Extract text content from a user message */
     private getUserMessageText;
+    private setEditorInputHistory;
+    private populateEditorInputHistoryFromMessages;
+    private populateEditorInputHistoryFromSession;
     private resetLiveTuiHistoryTrim;
     private clearPendingStreamingUiUpdate;
     private getSessionEntryCount;