@modular-prompt/driver 0.16.0 → 0.17.0

package/dist/local-inference/request-queue.js

@@ -1,6 +1,9 @@
 /**
  * LIP JSON-RPC リクエストキュー
  */
+function toError(error) {
+    return error instanceof Error ? error : new Error(String(error));
+}
 export class InferenceRequestQueue {
     queue = [];
     isProcessing = false;
@@ -79,7 +82,7 @@ export class InferenceRequestQueue {
             this.processNext();
         });
     }
-    addCachePrefillRequest(cachePath, messages, baseCachePath, trimToTokens, prefixOffsets, prefixHashes, tools, reasoningEffort) {
+    addCachePrefillRequest(cachePath, messages, baseCachePath, trimToTokens, prefixOffsets, prefixHashes, tools, reasoningEffort, images, maxImageSize) {
         return new Promise((resolve, reject) => {
             const request = {
                 method: 'cache_prefill',
@@ -90,6 +93,7 @@ export class InferenceRequestQueue {
                 ...(prefixOffsets && prefixHashes && { prefix_offsets: prefixOffsets, prefix_hashes: prefixHashes }),
                 ...(tools && { tools }),
                 ...(reasoningEffort && { reasoning_effort: reasoningEffort }),
+                ...(images?.length ? { images, maxImageSize } : {}),
             };
             this.queue.push({
                 request,
@@ -152,13 +156,28 @@ export class InferenceRequestQueue {
         this.isProcessing = true;
         const queueItem = this.queue[0];
         const { request, expectJsonResponse } = queueItem;
-        if (!expectJsonResponse) {
-            const stream = this.callbacks.createNewStream();
-            queueItem.resolve(stream);
-            this.queue.shift();
-        }
         const input = JSON.stringify(request);
-        this.callbacks.sendToProcess(input + '\n');
+        let stream;
+        try {
+            if (!expectJsonResponse) {
+                stream = this.callbacks.createNewStream();
+            }
+            this.callbacks.sendToProcess(input + '\n');
+            if (!expectJsonResponse && stream) {
+                queueItem.resolve(stream);
+                this.queue.shift();
+            }
+        }
+        catch (error) {
+            const requestError = toError(error);
+            if (stream) {
+                stream.destroy(requestError);
+            }
+            const failedItem = this.queue.shift();
+            failedItem?.reject?.(requestError);
+            this.isProcessing = false;
+            this.processNext();
+        }
     }
     handleJsonResponse(jsonData) {
         if (this.queue.length > 0) {

package/dist/local-inference/request-queue.js.map

@@ -1 +1 @@
-{"version":3,"file":"request-queue.js","sourceRoot":"","sources":["../../src/local-inference/request-queue.ts"],"names":[],"mappings":"AAAA;;GAEG;AAuCH,MAAM,OAAO,qBAAqB;IACxB,KAAK,GAAgB,EAAE,CAAC;IACxB,YAAY,GAAG,KAAK,CAAC;IACrB,SAAS,CAAwB;IACjC,kBAAkB,CAAwB;IAElD,YAAY,SAAgC,EAAE,kBAA0C;QACtF,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAA+C,CAAC,CAAC;IACjH,CAAC;IAED,sBAAsB;QACpB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,OAAO,GAAiC,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC;YACzE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;gBACd,OAAO;gBACP,OAAO;gBACP,MAAM;gBACN,kBAAkB,EAAE,IAAI;aACA,CAAC,CAAC;YAC5B,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,oBAAoB,CAClB,QAA4B,EAC5B,OAA6B;QAE7B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,OAAO,GAA+B;gBAC1C,MAAM,EAAE,aAAa;gBACrB,QAAQ;gBACR,OAAO;aACR,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;gBACd,OAAO;gBACP,OAAO;gBACP,MAAM;gBACN,kBAAkB,EAAE,IAAI;aACF,CAAC,CAAC;YAC1B,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,gBAAgB,CACd,QAA4B,EAC5B,OAAwD,EACxD,KAAiC,EACjC,eAA2C;QAE3C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,aAAa,GAAG,OAAO;gBAC3B,CAAC,CAAC;oBACE,GAAG,CAAC,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;oBAC3C,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACtD;gBACH,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,OAAO,GAAG;gBACd,MAAM,EAAE,QAAiB;gBACzB,QAAQ;gBACR,GAAG,CAAC,aAAa,IAAI,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7F,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;gBACvB,GAAG,CAAC,eAAe,IAAI,EAAE,gBAAgB,EAAE,eAAe,EAAE,CAAC;aAC9D,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;gBACd,OAAO;gBACP,OAAO;gBACP,MAAM;gBACN,kBAAkB,EAAE,IAAI;aACN,CAAC,CAAC;YACtB,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,kBAAkB,CAChB,QAA4B,EAC5B,KAAiC,EACjC,eAA2C;QAE3C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,OAAO,GAA6B;gBACxC,MAAM,EAAE,UAAU;gBAClB,QAAQ;gBACR,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;gBACvB,GAAG,CAAC,eAAe,IAAI,EAAE,gBAAgB,EAAE,eAAe,EAAE,CAAC;aAC9D,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;gBACd,OAAO;gBACP,OAAO;gBACP,MAAM;gBACN,kBAAkB,EAAE,IAAI;aACJ,CAAC,CAAC;YACxB,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,sBAAsB,CACpB,SAAiB,EACjB,QAA4B,EAC5B,aAAsB,EACtB,YAAqB,EACrB,aAAwB,EACxB,YAAuB,EACvB,KAAiC,EACjC,eAA2C;QAE3C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,OAAO,GAAiC;gBAC5C,MAAM,EAAE,eAAe;gBACvB,UAAU,EAAE,SAAS;gBACrB,QAAQ;gBACR,GAAG,CAAC,aAAa,IAAI,EAAE,eAAe,EAAE,aAAa,EAAE,CAAC;gBACxD,GAAG,CAAC,YAAY,IAAI,IAAI,IAAI,EAAE,cAAc,EAAE,YAAY,EAAE,CAAC;gBAC7D,GAAG,CAAC,aAAa,IAAI,YAAY,IAAI,EAAE,cAAc,EAAE,aAAa,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC;gBACpG,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;gBACvB,GAAG,CAAC,eAAe,IAAI,EAAE,gBAAgB,EAAE,eAAe,EAAE,CAAC;aAC9D,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;gBACd,OAAO;gBACP,OAAO;gBACP,MAAM;gBACN,kBAAkB,EAAE,IAAI;aACA,CAAC,CAAC;YAC5B,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,oBAAoB,CAClB,MAAc,EACd,OAAiB,EACjB,MAAiB,EACjB,YAAqB;QAErB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,CAAC;gBACH,MAAM,OAAO,GAA+B;oBAC1C,MAAM,EAAE,YAAY;oBACpB,MAAM;oBACN,OAAO,EAAE,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC;oBACzC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACpD,CAAC;gBACF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;oBACd,OAAO;oBACP,OAAO;oBACP,MAAM;iBACe,CAAC,CAAC;gBACzB,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,CAAC;YAChB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,kBAAkB,CAChB,MAAyB,EACzB,OAAiB,EACjB,MAAiB,EACjB,YAAqB,EACrB,SAAkB,EAClB,eAAwB,EACxB,MAAe;QAEf,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,CAAC;gBACH,MAAM,OAAO,GAA6B;oBACxC,MAAM,EAAE,UAAU;oBAClB,MAAM;oBACN,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC7B,OAAO,EAAE,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC;oBACzC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACnD,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC/C,GAAG,CAAC,eAAe,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAC3E,CAAC;gBACF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;oBACd,OAAO;oBACP,OAAO;oBACP,MAAM;iBACe,CAAC,CAAC;gBACzB,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,CAAC;YAChB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,WAAW;QACT,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjD,OAAO;QACT,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,EAAE,OAAO,EAAE,kBAAkB,EAAE,GAAG,SAAS,CAAC;QAElD,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACxB,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,eAAe,EAAE,CAAC;YAChD,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;IAC7C,CAAC;IAED,kBAAkB,CAAC,QAAgB;QACjC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACrC,IAAI,SAAS,EAAE,kBAAkB,EAAE,CAAC;gBAClC,IAAI,CAAC;oBACH,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;oBAC1C,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,aAAa,EAAE,CAAC;wBAC/C,IAAI,kBAAkB,IAAI,YAAY,EAAE,CAAC;4BACtC,SAAiC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;wBAC3D,CAAC;6BAAM,CAAC;4BACL,SAAiC,CAAC,OAAO,CAAC;gCACzC,gBAAgB,EAAE,IAAI;gCACtB,gBAAgB,EAAE,KAAK;gCACvB,yBAAyB,EAAE,IAAI;gCAC/B,KAAK,EAAE,YAAY,CAAC,KAAK,IAAI,gCAAgC;6BAC9D,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;yBAAM,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;wBACjD,IAAI,kBAAkB,IAAI,YAAY,IAAI,YAAY,CAAC,KAAK,EAAE,CAAC;4BAC5D,SAA6B,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;wBACvD,CAAC;6BAAM,CAAC;4BACL,SAA6B,CAAC,OAAO,CAAC;gCACrC,gBAAgB,EAAE,IAAI;gCACtB,KAAK,EAAE,YAAY,CAAC,KAAK,IAAI,2BAA2B;6BACzD,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;yBAAM,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;wBACnD,IAAI,aAAa,IAAI,YAAY,EAAE,CAAC;4BACjC,SAA+B,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;wBACzD,CAAC;6BAAM,CAAC;4BACL,SAA+B,CAAC,OAAO,CAAC;gCACvC,SAAS,EAAE,IAAI;gCACf,WAAW,EAAE,CAAC;gCACd,KAAK,EAAE,YAAY,CAAC,KAAK,IAAI,6BAA6B;6BAC3D,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;yBAAM,IAAI,YAAY,CAAC,KAAK,EAAE,CAAC;wBAC9B,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC;oBACpD,CAAC;yBAAM,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;wBACtD,SAAmC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;oBAC7D,CAAC;yBAAM,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,eAAe,EAAE,CAAC;wBACvD,SAAmC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;oBAC7D,CAAC;gBACH,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;wBAC/C,SAAmC,CAAC,OAAO,CAAC;4BAC3C,OAAO,EAAE,EAAE;4BACX,cAAc,EAAE,EAAE;4BAClB,QAAQ,EAAE,EAAE,mBAAmB,EAAE,KAAK,EAAE;yBACzC,CAAC,CAAC;oBACL,CAAC;yBAAM,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,aAAa,EAAE,CAAC;wBACrD,SAAiC,CAAC,OAAO,CAAC;4BACzC,gBAAgB,EAAE,IAAI;4BACtB,gBAAgB,EAAE,KAAK;4BACvB,yBAAyB,EAAE,IAAI;4BAC/B,KAAK,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;yBACxD,CAAC,CAAC;oBACL,CAAC;yBAAM,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;wBAChD,SAA6B,CAAC,OAAO,CAAC;4BACrC,gBAAgB,EAAE,IAAI;4BACtB,KAAK,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;yBACxD,CAAC,CAAC;oBACL,CAAC;yBAAM,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;wBAClD,SAA+B,CAAC,OAAO,CAAC;4BACvC,SAAS,EAAE,IAAI;4BACf,WAAW,EAAE,CAAC;4BACd,KAAK,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;yBACxD,CAAC,CAAC;oBACL,CAAC;yBAAM,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,eAAe,EAAE,CAAC;wBACvD,SAAmC,CAAC,MAAM,CACzC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAC9C,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,kBAAkB;QAChB,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,WAAW,EAAE,CAAC;IACrB,CAAC;IAED,mBAAmB;QACjB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE,CAAC;IACtC,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;IAED,SAAS,CAAC,KAAY;QACpB,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACrC,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC3B,IAAI,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;QACD,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;IAC5B,CAAC;CACF"}
+{"version":3,"file":"request-queue.js","sourceRoot":"","sources":["../../src/local-inference/request-queue.ts"],"names":[],"mappings":"AAAA;;GAEG;AAuCH,SAAS,OAAO,CAAC,KAAc;IAC7B,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AACnE,CAAC;AAED,MAAM,OAAO,qBAAqB;IACxB,KAAK,GAAgB,EAAE,CAAC;IACxB,YAAY,GAAG,KAAK,CAAC;IACrB,SAAS,CAAwB;IACjC,kBAAkB,CAAwB;IAElD,YAAY,SAAgC,EAAE,kBAA0C;QACtF,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAA+C,CAAC,CAAC;IACjH,CAAC;IAED,sBAAsB;QACpB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,OAAO,GAAiC,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC;YACzE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;gBACd,OAAO;gBACP,OAAO;gBACP,MAAM;gBACN,kBAAkB,EAAE,IAAI;aACA,CAAC,CAAC;YAC5B,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,oBAAoB,CAClB,QAA4B,EAC5B,OAA6B;QAE7B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,OAAO,GAA+B;gBAC1C,MAAM,EAAE,aAAa;gBACrB,QAAQ;gBACR,OAAO;aACR,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;gBACd,OAAO;gBACP,OAAO;gBACP,MAAM;gBACN,kBAAkB,EAAE,IAAI;aACF,CAAC,CAAC;YAC1B,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,gBAAgB,CACd,QAA4B,EAC5B,OAAwD,EACxD,KAAiC,EACjC,eAA2C;QAE3C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,aAAa,GAAG,OAAO;gBAC3B,CAAC,CAAC;oBACE,GAAG,CAAC,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;oBAC3C,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACtD;gBACH,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,OAAO,GAAG;gBACd,MAAM,EAAE,QAAiB;gBACzB,QAAQ;gBACR,GAAG,CAAC,aAAa,IAAI,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7F,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;gBACvB,GAAG,CAAC,eAAe,IAAI,EAAE,gBAAgB,EAAE,eAAe,EAAE,CAAC;aAC9D,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;gBACd,OAAO;gBACP,OAAO;gBACP,MAAM;gBACN,kBAAkB,EAAE,IAAI;aACN,CAAC,CAAC;YACtB,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,kBAAkB,CAChB,QAA4B,EAC5B,KAAiC,EACjC,eAA2C;QAE3C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,OAAO,GAA6B;gBACxC,MAAM,EAAE,UAAU;gBAClB,QAAQ;gBACR,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;gBACvB,GAAG,CAAC,eAAe,IAAI,EAAE,gBAAgB,EAAE,eAAe,EAAE,CAAC;aAC9D,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;gBACd,OAAO;gBACP,OAAO;gBACP,MAAM;gBACN,kBAAkB,EAAE,IAAI;aACJ,CAAC,CAAC;YACxB,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,sBAAsB,CACpB,SAAiB,EACjB,QAA4B,EAC5B,aAAsB,EACtB,YAAqB,EACrB,aAAwB,EACxB,YAAuB,EACvB,KAAiC,EACjC,eAA2C,EAC3C,MAAiB,EACjB,YAAqB;QAErB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,OAAO,GAAiC;gBAC5C,MAAM,EAAE,eAAe;gBACvB,UAAU,EAAE,SAAS;gBACrB,QAAQ;gBACR,GAAG,CAAC,aAAa,IAAI,EAAE,eAAe,EAAE,aAAa,EAAE,CAAC;gBACxD,GAAG,CAAC,YAAY,IAAI,IAAI,IAAI,EAAE,cAAc,EAAE,YAAY,EAAE,CAAC;gBAC7D,GAAG,CAAC,aAAa,IAAI,YAAY,IAAI,EAAE,cAAc,EAAE,aAAa,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC;gBACpG,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;gBACvB,GAAG,CAAC,eAAe,IAAI,EAAE,gBAAgB,EAAE,eAAe,EAAE,CAAC;gBAC7D,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACpD,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;gBACd,OAAO;gBACP,OAAO;gBACP,MAAM;gBACN,kBAAkB,EAAE,IAAI;aACA,CAAC,CAAC;YAC5B,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,oBAAoB,CAClB,MAAc,EACd,OAAiB,EACjB,MAAiB,EACjB,YAAqB;QAErB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,CAAC;gBACH,MAAM,OAAO,GAA+B;oBAC1C,MAAM,EAAE,YAAY;oBACpB,MAAM;oBACN,OAAO,EAAE,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC;oBACzC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACpD,CAAC;gBACF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;oBACd,OAAO;oBACP,OAAO;oBACP,MAAM;iBACe,CAAC,CAAC;gBACzB,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,CAAC;YAChB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,kBAAkB,CAChB,MAAyB,EACzB,OAAiB,EACjB,MAAiB,EACjB,YAAqB,EACrB,SAAkB,EAClB,eAAwB,EACxB,MAAe;QAEf,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,CAAC;gBACH,MAAM,OAAO,GAA6B;oBACxC,MAAM,EAAE,UAAU;oBAClB,MAAM;oBACN,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC7B,OAAO,EAAE,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC;oBACzC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACnD,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC/C,GAAG,CAAC,eAAe,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAC3E,CAAC;gBACF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;oBACd,OAAO;oBACP,OAAO;oBACP,MAAM;iBACe,CAAC,CAAC;gBACzB,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,CAAC;YAChB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,WAAW;QACT,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjD,OAAO;QACT,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,EAAE,OAAO,EAAE,kBAAkB,EAAE,GAAG,SAAS,CAAC;QAElD,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACtC,IAAI,MAA4B,CAAC;QACjC,IAAI,CAAC;YACH,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBACxB,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,eAAe,EAAE,CAAC;YAC5C,CAAC;YAED,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;YAE3C,IAAI,CAAC,kBAAkB,IAAI,MAAM,EAAE,CAAC;gBAClC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC1B,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACrB,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;YACpC,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;YAC/B,CAAC;YACD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACtC,UAAU,EAAE,MAAM,EAAE,CAAC,YAAY,CAAC,CAAC;YACnC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;YAC1B,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IAED,kBAAkB,CAAC,QAAgB;QACjC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACrC,IAAI,SAAS,EAAE,kBAAkB,EAAE,CAAC;gBAClC,IAAI,CAAC;oBACH,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;oBAC1C,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,aAAa,EAAE,CAAC;wBAC/C,IAAI,kBAAkB,IAAI,YAAY,EAAE,CAAC;4BACtC,SAAiC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;wBAC3D,CAAC;6BAAM,CAAC;4BACL,SAAiC,CAAC,OAAO,CAAC;gCACzC,gBAAgB,EAAE,IAAI;gCACtB,gBAAgB,EAAE,KAAK;gCACvB,yBAAyB,EAAE,IAAI;gCAC/B,KAAK,EAAE,YAAY,CAAC,KAAK,IAAI,gCAAgC;6BAC9D,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;yBAAM,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;wBACjD,IAAI,kBAAkB,IAAI,YAAY,IAAI,YAAY,CAAC,KAAK,EAAE,CAAC;4BAC5D,SAA6B,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;wBACvD,CAAC;6BAAM,CAAC;4BACL,SAA6B,CAAC,OAAO,CAAC;gCACrC,gBAAgB,EAAE,IAAI;gCACtB,KAAK,EAAE,YAAY,CAAC,KAAK,IAAI,2BAA2B;6BACzD,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;yBAAM,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;wBACnD,IAAI,aAAa,IAAI,YAAY,EAAE,CAAC;4BACjC,SAA+B,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;wBACzD,CAAC;6BAAM,CAAC;4BACL,SAA+B,CAAC,OAAO,CAAC;gCACvC,SAAS,EAAE,IAAI;gCACf,WAAW,EAAE,CAAC;gCACd,KAAK,EAAE,YAAY,CAAC,KAAK,IAAI,6BAA6B;6BAC3D,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;yBAAM,IAAI,YAAY,CAAC,KAAK,EAAE,CAAC;wBAC9B,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC;oBACpD,CAAC;yBAAM,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;wBACtD,SAAmC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;oBAC7D,CAAC;yBAAM,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,eAAe,EAAE,CAAC;wBACvD,SAAmC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;oBAC7D,CAAC;gBACH,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;wBAC/C,SAAmC,CAAC,OAAO,CAAC;4BAC3C,OAAO,EAAE,EAAE;4BACX,cAAc,EAAE,EAAE;4BAClB,QAAQ,EAAE,EAAE,mBAAmB,EAAE,KAAK,EAAE;yBACzC,CAAC,CAAC;oBACL,CAAC;yBAAM,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,aAAa,EAAE,CAAC;wBACrD,SAAiC,CAAC,OAAO,CAAC;4BACzC,gBAAgB,EAAE,IAAI;4BACtB,gBAAgB,EAAE,KAAK;4BACvB,yBAAyB,EAAE,IAAI;4BAC/B,KAAK,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;yBACxD,CAAC,CAAC;oBACL,CAAC;yBAAM,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;wBAChD,SAA6B,CAAC,OAAO,CAAC;4BACrC,gBAAgB,EAAE,IAAI;4BACtB,KAAK,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;yBACxD,CAAC,CAAC;oBACL,CAAC;yBAAM,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;wBAClD,SAA+B,CAAC,OAAO,CAAC;4BACvC,SAAS,EAAE,IAAI;4BACf,WAAW,EAAE,CAAC;4BACd,KAAK,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;yBACxD,CAAC,CAAC;oBACL,CAAC;yBAAM,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,eAAe,EAAE,CAAC;wBACvD,SAAmC,CAAC,MAAM,CACzC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAC9C,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,kBAAkB;QAChB,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,WAAW,EAAE,CAAC;IACrB,CAAC;IAED,mBAAmB;QACjB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE,CAAC;IACtC,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;IAED,SAAS,CAAC,KAAY;QACpB,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACrC,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC3B,IAAI,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;QACD,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;IAC5B,CAAC;CACF"}

package/dist/local-inference/stream-utils.d.ts

@@ -2,6 +2,12 @@ import { Readable } from 'stream';
 export interface StreamMeta {
     prompt_tokens?: number;
     generation_tokens?: number;
+    /** Whether the requested prompt cache was actually loaded for this stream. */
+    cache_loaded?: boolean;
+    /** Tokens read from a backend-owned prompt/KV cache for this stream. */
+    cache_read_tokens?: number;
+    /** Tokens written to a backend-owned prompt/KV cache for this stream. */
+    cache_write_tokens?: number;
 }
 export declare const META_MARKER = "\u001E__META__:";
 export declare function extractStreamMeta(content: string): {

package/dist/local-inference/stream-utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stream-utils.d.ts","sourceRoot":"","sources":["../../src/local-inference/stream-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAElC,MAAM,WAAW,UAAU;IACzB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,eAAO,MAAM,WAAW,oBAAkB,CAAC;AAE3C,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,UAAU,CAAA;CAAE,CASxF;AAED,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,QAAQ,EAChB,SAAS,CAAC,EAAE,MAAM,OAAO,GACxB;IACD,QAAQ,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;IAChC,UAAU,EAAE,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,UAAU,CAAC;QAAC,KAAK,EAAE,KAAK,GAAG,IAAI,CAAA;KAAE,CAAC,CAAC;CACjF,CA0DA"}
+{"version":3,"file":"stream-utils.d.ts","sourceRoot":"","sources":["../../src/local-inference/stream-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAElC,MAAM,WAAW,UAAU;IACzB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,8EAA8E;IAC9E,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,wEAAwE;IACxE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,yEAAyE;IACzE,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,eAAO,MAAM,WAAW,oBAAkB,CAAC;AAE3C,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,UAAU,CAAA;CAAE,CASxF;AAED,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,QAAQ,EAChB,SAAS,CAAC,EAAE,MAAM,OAAO,GACxB;IACD,QAAQ,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;IAChC,UAAU,EAAE,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,UAAU,CAAC;QAAC,KAAK,EAAE,KAAK,GAAG,IAAI,CAAA;KAAE,CAAC,CAAC;CACjF,CA0DA"}

package/dist/local-inference/stream-utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"stream-utils.js","sourceRoot":"","sources":["../../src/local-inference/stream-utils.ts"],"names":[],"mappings":"AAOA,MAAM,CAAC,MAAM,WAAW,GAAG,eAAe,CAAC;AAE3C,MAAM,UAAU,iBAAiB,CAAC,OAAe;IAC/C,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;IAC7C,IAAI,GAAG,KAAK,CAAC,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IAC7C,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IACxD,IAAI,CAAC;QACH,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;IACvE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IACtD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,MAAgB,EAChB,SAAyB;IAKzB,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,iBAA8F,CAAC;IACnG,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,MAAM,MAAM,GAAG,CAAC,KAAmB,EAAE,EAAE;QACrC,IAAI,OAAO;YAAE,OAAO;QACpB,OAAO,GAAG,IAAI,CAAC;QACf,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5B,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC;QACjD,MAAM,OAAO,GAAG,SAAS,EAAE,EAAE,IAAI,KAAK,CAAC;QACvC,iBAAiB,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;IACtE,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,IAAI,OAAO,CAA6D,CAAC,OAAO,EAAE,EAAE;QACrG,iBAAiB,GAAG,OAAO,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG;QACf,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;YAC3B,IAAI,CAAC;gBACH,IAAI,MAAM,GAAG,EAAE,CAAC;gBAChB,IAAI,WAAW,GAAG,KAAK,CAAC;gBACxB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;oBACjC,IAAI,SAAS,EAAE,EAAE,EAAE,CAAC;wBAClB,MAAM;oBACR,CAAC;oBACD,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC;oBAC7B,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACjB,IAAI,WAAW;wBAAE,SAAS;oBAC1B,MAAM,IAAI,GAAG,CAAC;oBACd,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;oBAC9C,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;wBACrB,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;wBACxC,IAAI,IAAI;4BAAE,MAAM,IAAI,CAAC;wBACrB,WAAW,GAAG,IAAI,CAAC;oBACrB,CAAC;yBAAM,CAAC;wBACN,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;wBACzD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;4BAChB,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;4BAC/B,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;wBACjC,CAAC;oBACH,CAAC;gBACH,CAAC;gBACD,IAAI,CAAC,WAAW,IAAI,MAAM;oBAAE,MAAM,MAAM,CAAC;YAC3C,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAc,CAAC,CAAC;gBACvB,MAAM,OAAO,GAAG,SAAS,EAAE,EAAE,IAAI,KAAK,CAAC;gBACvC,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC;oBAAS,CAAC;gBACT,MAAM,CAAC,IAAI,CAAC,CAAC;YACf,CAAC;QACH,CAAC;KACF,CAAC;IAEF,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC;AAClC,CAAC"}
+{"version":3,"file":"stream-utils.js","sourceRoot":"","sources":["../../src/local-inference/stream-utils.ts"],"names":[],"mappings":"AAaA,MAAM,CAAC,MAAM,WAAW,GAAG,eAAe,CAAC;AAE3C,MAAM,UAAU,iBAAiB,CAAC,OAAe;IAC/C,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;IAC7C,IAAI,GAAG,KAAK,CAAC,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IAC7C,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IACxD,IAAI,CAAC;QACH,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;IACvE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IACtD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,MAAgB,EAChB,SAAyB;IAKzB,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,iBAA8F,CAAC;IACnG,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,MAAM,MAAM,GAAG,CAAC,KAAmB,EAAE,EAAE;QACrC,IAAI,OAAO;YAAE,OAAO;QACpB,OAAO,GAAG,IAAI,CAAC;QACf,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5B,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC;QACjD,MAAM,OAAO,GAAG,SAAS,EAAE,EAAE,IAAI,KAAK,CAAC;QACvC,iBAAiB,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;IACtE,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,IAAI,OAAO,CAA6D,CAAC,OAAO,EAAE,EAAE;QACrG,iBAAiB,GAAG,OAAO,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG;QACf,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;YAC3B,IAAI,CAAC;gBACH,IAAI,MAAM,GAAG,EAAE,CAAC;gBAChB,IAAI,WAAW,GAAG,KAAK,CAAC;gBACxB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;oBACjC,IAAI,SAAS,EAAE,EAAE,EAAE,CAAC;wBAClB,MAAM;oBACR,CAAC;oBACD,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC;oBAC7B,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACjB,IAAI,WAAW;wBAAE,SAAS;oBAC1B,MAAM,IAAI,GAAG,CAAC;oBACd,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;oBAC9C,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;wBACrB,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;wBACxC,IAAI,IAAI;4BAAE,MAAM,IAAI,CAAC;wBACrB,WAAW,GAAG,IAAI,CAAC;oBACrB,CAAC;yBAAM,CAAC;wBACN,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;wBACzD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;4BAChB,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;4BAC/B,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;wBACjC,CAAC;oBACH,CAAC;gBACH,CAAC;gBACD,IAAI,CAAC,WAAW,IAAI,MAAM;oBAAE,MAAM,MAAM,CAAC;YAC3C,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAc,CAAC,CAAC;gBACvB,MAAM,OAAO,GAAG,SAAS,EAAE,EAAE,IAAI,KAAK,CAAC;gBACvC,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC;oBAAS,CAAC;gBACT,MAAM,CAAC,IAAI,CAAC,CAAC;YACf,CAAC;QACH,CAAC;KACF,CAAC;IAEF,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC;AAClC,CAAC"}

package/dist/mlx-ml/mlx-cache-controller.d.ts

@@ -15,6 +15,7 @@ export declare class MlxCacheController implements PromptCacheController {
     private cacheDirReady;
     private closed;
     private bound;
+    private modelKind;
     private cleanupHandler?;
     private messageProcessor?;
     private formatterOptions;
@@ -25,8 +26,12 @@ export declare class MlxCacheController implements PromptCacheController {
     private lastHandleToolsHash?;
     private lastHandleReasoningEffort?;
     private cacheIndex;
+    /** Token counts returned by the backend, including VLM exact disk refs. */
+    private cacheTokenCounts;
     private stats;
     constructor(options?: MlxCacheControllerOptions);
+    /** Select backend-local storage before the controller is bound. */
+    setModelKind(modelKind?: 'lm' | 'vlm'): void;
     bind(process: MlxProcess, formatterOptions: FormatterOptions, messageProcessor?: (messages: MlxMessage[]) => MlxMessage[]): Promise<void>;
     private ensureCacheDir;
     private get indexPath();
@@ -35,6 +40,7 @@ export declare class MlxCacheController implements PromptCacheController {
     private saveIndex;
     private computeFormatterOptionsHash;
     private computeToolsHash;
+    private computeImageCacheIdentity;
     private updateLastCache;
     private clearLastCache;
     private computeElementHashes;
@@ -45,6 +51,9 @@ export declare class MlxCacheController implements PromptCacheController {
     private addToIndex;
     private computeCacheKey;
     private generateCachePath;
+    private getIndexedCachePath;
+    private getEntryCachePath;
+    private toIndexCachePath;
     recordQuery(): void;
     recordPromptTokens(newPromptTokens: number, cacheTokensUsed: number): void;
     readCacheTokenCount(cachePath: string): number;

package/dist/mlx-ml/mlx-cache-controller.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mlx-cache-controller.d.ts","sourceRoot":"","sources":["../../src/mlx-ml/mlx-cache-controller.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAErG,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAG9D,OAAO,KAAK,EAAE,UAAU,EAAqB,MAAM,oBAAoB,CAAC;AACxE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAuBrD,MAAM,WAAW,yBAAyB;IACxC,2CAA2C;IAC3C,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAWD,qBAAa,kBAAmB,YAAW,qBAAqB;IAC9D,OAAO,CAAC,WAAW,CAAkC;IACrD,OAAO,CAAC,gBAAgB,CAA2C;IACnE,OAAO,CAAC,OAAO,CAAC,CAAa;IAC7B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,UAAU,CAAU;IAC5B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,cAAc,CAAC,CAAa;IACpC,OAAO,CAAC,gBAAgB,CAAC,CAA2C;IACpE,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,UAAU,CAAC,CAAc;IACjC,OAAO,CAAC,iBAAiB,CAAC,CAAW;IACrC,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,8BAA8B,CAAC,CAAS;IAChD,OAAO,CAAC,mBAAmB,CAAC,CAAS;IACrC,OAAO,CAAC,yBAAyB,CAAC,CAAS;IAC3C,OAAO,CAAC,UAAU,CAA2C;IAC7D,OAAO,CAAC,KAAK,CAOX;gBAEU,OAAO,CAAC,EAAE,yBAAyB;IAWzC,IAAI,CACR,OAAO,EAAE,UAAU,EACnB,gBAAgB,EAAE,gBAAgB,EAClC,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,UAAU,EAAE,GAC1D,OAAO,CAAC,IAAI,CAAC;YAsBF,cAAc;IAM5B,OAAO,KAAK,SAAS,GAEpB;IAED,OAAO,CAAC,kBAAkB;YAUZ,SAAS;YAkBT,SAAS;IAevB,OAAO,CAAC,2BAA2B;IAOnC,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,eAAe;IASvB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,oBAAoB;IAW5B,OAAO,CAAC,cAAc;IAetB,OAAO,CAAC,sBAAsB;YAMhB,iBAAiB;YAmFjB,YAAY;IAgI1B,OAAO,CAAC,UAAU;IAelB,OAAO,CAAC,eAAe;IAoBvB,OAAO,CAAC,iBAAiB;IAIzB,WAAW,IAAI,IAAI;IAInB,kBAAkB,CAAC,eAAe,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,GAAG,IAAI;IAK1E,mBAAmB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM;IAI9C,QAAQ;;;;;;;;IAWF,OAAO,CAAC,MAAM,EAAE,kBAAkB,GAAG,OAAO,CAAC,WAAW,CAAC;IAsC/D,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAElC;YAEY,WAAW;IAoJzB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAmBpB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CA6B7B"}
+{"version":3,"file":"mlx-cache-controller.d.ts","sourceRoot":"","sources":["../../src/mlx-ml/mlx-cache-controller.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAErG,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAG9D,OAAO,KAAK,EAAE,UAAU,EAAqB,MAAM,oBAAoB,CAAC;AACxE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAmCrD,MAAM,WAAW,yBAAyB;IACxC,2CAA2C;IAC3C,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAWD,qBAAa,kBAAmB,YAAW,qBAAqB;IAC9D,OAAO,CAAC,WAAW,CAAkC;IACrD,OAAO,CAAC,gBAAgB,CAA2C;IACnE,OAAO,CAAC,OAAO,CAAC,CAAa;IAC7B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,UAAU,CAAU;IAC5B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,SAAS,CAAsB;IACvC,OAAO,CAAC,cAAc,CAAC,CAAa;IACpC,OAAO,CAAC,gBAAgB,CAAC,CAA2C;IACpE,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,UAAU,CAAC,CAAc;IACjC,OAAO,CAAC,iBAAiB,CAAC,CAAW;IACrC,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,8BAA8B,CAAC,CAAS;IAChD,OAAO,CAAC,mBAAmB,CAAC,CAAS;IACrC,OAAO,CAAC,yBAAyB,CAAC,CAAS;IAC3C,OAAO,CAAC,UAAU,CAA2C;IAC7D,2EAA2E;IAC3E,OAAO,CAAC,gBAAgB,CAA6B;IACrD,OAAO,CAAC,KAAK,CAOX;gBAEU,OAAO,CAAC,EAAE,yBAAyB;IAa/C,mEAAmE;IACnE,YAAY,CAAC,SAAS,CAAC,EAAE,IAAI,GAAG,KAAK,GAAG,IAAI;IAOtC,IAAI,CACR,OAAO,EAAE,UAAU,EACnB,gBAAgB,EAAE,gBAAgB,EAClC,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,UAAU,EAAE,GAC1D,OAAO,CAAC,IAAI,CAAC;YAsBF,cAAc;IAM5B,OAAO,KAAK,SAAS,GAEpB;IAED,OAAO,CAAC,kBAAkB;YAUZ,SAAS;YAkBT,SAAS;IAevB,OAAO,CAAC,2BAA2B;IAOnC,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,yBAAyB;IAyBjC,OAAO,CAAC,eAAe;IASvB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,oBAAoB;IAW5B,OAAO,CAAC,cAAc;IAetB,OAAO,CAAC,sBAAsB;YAMhB,iBAAiB;YAmFjB,YAAY;IAuI1B,OAAO,CAAC,UAAU;IA4BlB,OAAO,CAAC,eAAe;IA2BvB,OAAO,CAAC,iBAAiB;IAOzB,OAAO,CAAC,mBAAmB;IAQ3B,OAAO,CAAC,iBAAiB;IAOzB,OAAO,CAAC,gBAAgB;IAcxB,WAAW,IAAI,IAAI;IAInB,kBAAkB,CAAC,eAAe,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,GAAG,IAAI;IAK1E,mBAAmB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM;IAI9C,QAAQ;;;;;;;;IAWF,OAAO,CAAC,MAAM,EAAE,kBAAkB,GAAG,OAAO,CAAC,WAAW,CAAC;IAsC/D,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAElC;YAEY,WAAW;IA0LzB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAmBpB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CA8B7B"}

package/dist/mlx-ml/mlx-cache-controller.js

@@ -1,6 +1,6 @@
 import { createHash, randomBytes } from 'node:crypto';
 import { tmpdir } from 'node:os';
-import { join } from 'node:path';
+import { isAbsolute, join, relative, resolve as resolvePath, sep } from 'node:path';
 import { rmSync, existsSync, readFileSync } from 'node:fs';
 import { unlink, mkdir, rm, writeFile, readFile } from 'node:fs/promises';
 import { lock as lockFile } from 'proper-lockfile';
@@ -8,7 +8,15 @@ import { formatPromptAsMessages } from '../formatter/converter.js';
 import { convertMessages, convertToolDefinitions } from './mlx-message-utils.js';
 import { Logger } from '@modular-prompt/utils';
 const logger = new Logger({ prefix: 'MLX', context: 'cache' });
-const CACHE_FILE_EXTENSION = '.safetensors.zip';
+const CACHE_FILE_EXTENSIONS = {
+    lm: '.safetensors.zip',
+    // mlx-vlm's exact_cache_v1 snapshot is a standalone safetensors file and
+    // must never be mistaken for an mlx-lm archive.
+    vlm: '.vlm.safetensors',
+    // Image-bearing VLM snapshots have a separate namespace and sidecar
+    // layout.  They are intentionally incompatible with text-only VLM refs.
+    vlmVision: '.vlm-vision.safetensors',
+};
 export class MlxCacheController {
     cacheByHash = new Map();
     inflightRequests = new Map();
@@ -18,6 +26,7 @@ export class MlxCacheController {
     cacheDirReady = false;
     closed = false;
     bound = false;
+    modelKind = 'lm';
     cleanupHandler;
     messageProcessor;
     formatterOptions;
@@ -28,6 +37,8 @@ export class MlxCacheController {
     lastHandleToolsHash;
     lastHandleReasoningEffort;
     cacheIndex = { version: 1, entries: [] };
+    /** Token counts returned by the backend, including VLM exact disk refs. */
+    cacheTokenCounts = new Map();
     stats = {
         totalQueries: 0,
         memoryHit: 0, diskHit: 0, incremental: 0, fresh: 0,
@@ -39,7 +50,9 @@ export class MlxCacheController {
     constructor(options) {
         this.formatterOptions = {};
         if (options?.cacheDir) {
-            this.cacheDir = options.cacheDir;
+            // Keep backend refs and relative index paths stable when callers pass a
+            // relative cacheDir (the Python process may have a different cwd).
+            this.cacheDir = resolvePath(options.cacheDir);
             this.managedDir = false;
         }
         else {
@@ -47,6 +60,13 @@ export class MlxCacheController {
             this.managedDir = true;
         }
     }
+    /** Select backend-local storage before the controller is bound. */
+    setModelKind(modelKind) {
+        if (this.bound) {
+            throw new Error('MlxCacheController model kind must be set before bind');
+        }
+        this.modelKind = modelKind === 'vlm' ? 'vlm' : 'lm';
+    }
     async bind(process, formatterOptions, messageProcessor) {
         if (this.bound) {
             throw new Error('MlxCacheController is already bound to a process');
@@ -139,6 +159,28 @@ export class MlxCacheController {
         const sorted = [...tools].sort((a, b) => a.name.localeCompare(b.name));
         return createHash('sha256').update(JSON.stringify(sorted)).digest('hex');
     }
+    computeImageCacheIdentity(images, maxImageSize) {
+        if (!images || images.length === 0)
+            return undefined;
+        return {
+            layout: 'vision_cache_v1',
+            maxImageSize: maxImageSize ?? 768,
+            images: images.map((image) => {
+                let contentHash;
+                try {
+                    // Include the source bytes in the controller key when the source is
+                    // a local file.  The Python sidecar remains the authoritative
+                    // normalized-image/APC extra_hash check at load time.
+                    contentHash = createHash('sha256').update(readFileSync(image)).digest('hex');
+                }
+                catch {
+                    // URLs and unresolved paths are still distinguished by their ref;
+                    // mlx-vlm will reject an unreadable path during prefill/load.
+                }
+                return { ref: image, contentHash };
+            }),
+        };
+    }
     updateLastCache(handle, elementHashes, params) {
         this.lastHandle = handle;
         this.lastElementHashes = elementHashes;
@@ -225,7 +267,7 @@ export class MlxCacheController {
                 output: [],
             };
             const chatMessages = formatPromptAsMessages(partialPrompt, this.formatterOptions);
-            let mlxMessages = convertMessages(chatMessages);
+            let mlxMessages = convertMessages(chatMessages, this.modelKind === 'vlm');
             if (this.messageProcessor) {
                 mlxMessages = this.messageProcessor(mlxMessages);
             }
@@ -256,6 +298,11 @@ export class MlxCacheController {
         return { offsets, hashes };
     }
     async findBestBase(params, fullTokens) {
+        // exact_cache_v1 VLM snapshots are intentionally not used for incremental
+        // prefill.  A VLM cache is still eligible for an exact disk hit in
+        // createCache(), but base_cache_path/trim_to_tokens remain Phase 3 work.
+        if (this.modelKind === 'vlm')
+            return undefined;
         const newHashes = this.computeElementHashes(params);
         if (newHashes.length === 0)
             return undefined;
@@ -264,6 +311,8 @@ export class MlxCacheController {
         const candidates = [];
         const staleKeys = [];
         for (const entry of this.cacheIndex.entries) {
+            if ((entry.backend ?? 'lm') !== 'lm')
+                continue;
             if (entry.hint === 'release')
                 continue;
             if (entry.model !== params.model || entry.formatterOptionsHash !== fmtHash)
@@ -272,7 +321,7 @@ export class MlxCacheController {
                 continue;
             if ((entry.reasoningEffort ?? '') !== (params.reasoningEffort ?? ''))
                 continue;
-            const path = this.generateCachePath(entry.key);
+            const path = entry.path ?? this.generateCachePath(entry.key);
             if (existsSync(path) && this.readMetaTokenCount(path) > 0) {
                 candidates.push({ path, elementHashes: entry.elementHashes, label: entry.key.slice(0, 8) });
             }
@@ -294,7 +343,7 @@ export class MlxCacheController {
             }
         }
         if (staleKeys.length > 0) {
-            this.cacheIndex.entries = this.cacheIndex.entries.filter(e => !staleKeys.includes(e.key));
+            this.cacheIndex.entries = this.cacheIndex.entries.filter(e => !(staleKeys.includes(e.key) && (e.backend ?? 'lm') === 'lm'));
             this.saveIndex().catch(() => { });
         }
         if (candidates.length === 0)
@@ -364,10 +413,18 @@ export class MlxCacheController {
         }
         return bestInfo;
     }
-    addToIndex(params, cacheKey) {
-        // avoid duplicates
-        if (this.cacheIndex.entries.some(e => e.key === cacheKey))
+    addToIndex(params, cacheKey, cachePath) {
+        // avoid duplicates while refreshing the actual backend path after a VLM
+        // prefill (the logical path is only the namespace seed).
+        const existing = this.cacheIndex.entries.find(e => e.key === cacheKey && (e.backend ?? 'lm') === this.modelKind);
+        if (existing) {
+            existing.backend = this.modelKind;
+            if (this.modelKind === 'vlm') {
+                existing.path = this.toIndexCachePath(cachePath);
+            }
+            existing.hint = undefined;
             return;
+        }
         this.cacheIndex.entries.push({
             key: cacheKey,
             model: params.model,
@@ -376,6 +433,8 @@ export class MlxCacheController {
             toolsHash: this.computeToolsHash(params.tools),
             reasoningEffort: params.reasoningEffort,
             createdAt: new Date().toISOString(),
+            backend: this.modelKind,
+            path: this.modelKind === 'vlm' ? this.toIndexCachePath(cachePath) : undefined,
         });
     }
     computeCacheKey(params) {
@@ -395,10 +454,42 @@ export class MlxCacheController {
         if (params.reasoningEffort) {
             payload.reasoningEffort = params.reasoningEffort;
         }
+        const imageIdentity = this.computeImageCacheIdentity(params.images, params.maxImageSize);
+        if (imageIdentity) {
+            payload.imageCache = imageIdentity;
+        }
         return createHash('sha256').update(JSON.stringify(payload)).digest('hex');
     }
-    generateCachePath(cacheKey) {
-        return join(this.cacheDir, `${cacheKey}${CACHE_FILE_EXTENSION}`);
+    generateCachePath(cacheKey, vision = false) {
+        const extension = vision
+            ? CACHE_FILE_EXTENSIONS.vlmVision
+            : CACHE_FILE_EXTENSIONS[this.modelKind];
+        return join(this.cacheDir, `${cacheKey}${extension}`);
+    }
+    getIndexedCachePath(cacheKey) {
+        const indexedPath = this.cacheIndex.entries.find(e => e.key === cacheKey && (e.backend ?? 'lm') === this.modelKind)?.path;
+        if (!indexedPath)
+            return undefined;
+        return isAbsolute(indexedPath) ? indexedPath : join(this.cacheDir, indexedPath);
+    }
+    getEntryCachePath(entry) {
+        if (entry.path) {
+            return isAbsolute(entry.path) ? entry.path : join(this.cacheDir, entry.path);
+        }
+        return this.generateCachePath(entry.key);
+    }
+    toIndexCachePath(cachePath) {
+        if (this.modelKind !== 'vlm') {
+            return cachePath;
+        }
+        const absoluteCachePath = isAbsolute(cachePath) ? cachePath : resolvePath(cachePath);
+        const relativePath = relative(this.cacheDir, absoluteCachePath);
+        // Backend-created VLM paths are inside cacheDir. Keep an absolute path
+        // only for an incompatible/custom backend result that cannot be relocated.
+        if (relativePath === '..' || relativePath.startsWith(`..${sep}`)) {
+            return cachePath;
+        }
+        return relativePath;
     }
     recordQuery() {
         this.stats.totalQueries++;
@@ -408,7 +499,7 @@ export class MlxCacheController {
         this.stats.totalCacheTokensUsed += cacheTokensUsed;
     }
     readCacheTokenCount(cachePath) {
-        return this.readMetaTokenCount(cachePath);
+        return this.cacheTokenCounts.get(cachePath) ?? this.readMetaTokenCount(cachePath);
     }
     getStats() {
         const s = this.stats;
@@ -463,12 +554,25 @@ export class MlxCacheController {
             logger.verbose('cache dir creation failed, skipping cache:', e instanceof Error ? e.message : String(e));
             return MlxCacheController.EMPTY_HANDLE;
         }
-        const cachePath = this.generateCachePath(cacheKey);
+        const visionCache = this.modelKind === 'vlm' && (params.images?.length ?? 0) > 0;
+        // VLM prefill returns the actual APC snapshot path.  Reuse that path when
+        // a fixed-cache index has it; otherwise start with a deterministic logical
+        // path that the Python backend turns into a DiskBlockStore namespace.
+        const indexedPath = this.getIndexedCachePath(cacheKey);
+        const cachePath = indexedPath
+            && existsSync(indexedPath)
+            && existsSync(indexedPath + '.meta.json')
+            && this.readMetaTokenCount(indexedPath) > 0
+            ? indexedPath
+            : this.generateCachePath(cacheKey, visionCache);
+        let effectiveCachePath = cachePath;
         const elementHashes = this.computeElementHashes(params);
         let supersededRef;
-        // Disk hit check (exact same cache already exists)
+        // Disk hit check (exact same cache already exists).  Both LM archives and
+        // VLM exact_cache_v1 snapshots use the same sidecar token-count contract.
         if (existsSync(cachePath) && existsSync(cachePath + '.meta.json') && this.readMetaTokenCount(cachePath) > 0) {
             this.stats.diskHit++;
+            this.cacheTokenCounts.set(cachePath, this.readMetaTokenCount(cachePath));
             logger.verbose('reusing existing cache file', cacheKey.slice(0, 12));
         }
         else {
@@ -479,26 +583,34 @@ export class MlxCacheController {
                 output: [],
             };
             const chatMessages = formatPromptAsMessages(prefillPrompt, this.formatterOptions);
-            const preMergeMessages = convertMessages(chatMessages);
+            const preMergeMessages = convertMessages(chatMessages, this.modelKind === 'vlm');
             let mlxMessages = preMergeMessages;
             if (this.messageProcessor) {
                 mlxMessages = this.messageProcessor(mlxMessages);
             }
             const hasTools = params.tools && params.tools.length > 0;
             const mlxTools = hasTools ? convertToolDefinitions(params.tools) : undefined;
-            // Tokenize to get full token IDs (for findBestBase + prefix computation)
+            // Tokenize to get full token IDs (for findBestBase + prefix computation).
+            // Image-bearing VLM prompts are deliberately excluded: the shared
+            // tokenize protocol carries messages but not image sources, so it
+            // cannot reproduce mlx-vlm's dynamic image-token expansion.  The VLM
+            // backend computes the authoritative image-aware IDs during prefill.
             let fullTokens = null;
-            try {
-                const tokenResult = await this.process.tokenize(mlxMessages, mlxTools, params.reasoningEffort);
-                if (!tokenResult.error && tokenResult.token_ids) {
-                    fullTokens = tokenResult.token_ids;
+            if (!visionCache) {
+                try {
+                    const tokenResult = await this.process.tokenize(mlxMessages, mlxTools, params.reasoningEffort);
+                    if (!tokenResult.error && tokenResult.token_ids) {
+                        fullTokens = tokenResult.token_ids;
+                    }
+                }
+                catch {
+                    // tokenize failure — proceed without prefix matching
                 }
             }
-            catch {
-                // tokenize failure — proceed without prefix matching
-            }
-            // Find best base cache
-            const base = fullTokens
+            // Find best base cache.  VLM exact snapshots intentionally do not
+            // participate in incremental prefill until a safe trim/replay protocol
+            // is implemented.
+            const base = this.modelKind === 'lm' && fullTokens
                 ? await this.findBestBase(params, fullTokens)
                 : undefined;
             if (base?.coversAll) {
@@ -537,14 +649,26 @@ export class MlxCacheController {
             logger.debug('prefill', cachePath);
             const prefillStart = performance.now();
             try {
-                await this.process.cachePrefill(cachePath, mlxMessages, base?.path, base?.trimTokens, prefixOffsets, prefixHashes, mlxTools, params.reasoningEffort);
+                const prefillResult = await this.process.cachePrefill(cachePath, mlxMessages, base?.path, base?.trimTokens, prefixOffsets, prefixHashes, mlxTools, params.reasoningEffort, params.images, params.maxImageSize);
+                const returnedTokenCount = typeof prefillResult?.token_count === 'number'
+                    ? prefillResult.token_count
+                    : undefined;
+                if (this.modelKind === 'vlm'
+                    && typeof prefillResult?.cache_path === 'string'
+                    && prefillResult.cache_path.length > 0) {
+                    effectiveCachePath = prefillResult.cache_path;
+                }
+                const tokenCount = returnedTokenCount ?? this.readMetaTokenCount(effectiveCachePath);
+                if (tokenCount > 0) {
+                    this.cacheTokenCounts.set(effectiveCachePath, tokenCount);
+                }
             }
             catch (e) {
                 logger.verbose('prefill failed, skipping cache:', e instanceof Error ? e.message : String(e));
                 return MlxCacheController.EMPTY_HANDLE;
             }
             const prefillMs = performance.now() - prefillStart;
-            const newTokens = this.readMetaTokenCount(cachePath);
+            const newTokens = this.readCacheTokenCount(effectiveCachePath);
             this.stats.prefillTokens += newTokens;
             if (base) {
                 this.stats.incremental++;
@@ -556,13 +680,13 @@ export class MlxCacheController {
             }
             logger.verbose(`prefill ${prefillMs.toFixed(0)}ms`, base ? '(incremental)' : '(fresh)');
             if (this.closed) {
-                await unlink(cachePath).catch(() => { });
-                await unlink(cachePath + '.meta.json').catch(() => { });
+                await unlink(effectiveCachePath).catch(() => { });
+                await unlink(effectiveCachePath + '.meta.json').catch(() => { });
                 return MlxCacheController.EMPTY_HANDLE;
             }
         }
         const handle = {
-            ref: cachePath,
+            ref: effectiveCachePath,
             includes: {
                 instructions: (params.instructions?.length ?? 0) > 0,
                 dataElementCount: params.data?.length ?? 0,
@@ -572,7 +696,7 @@ export class MlxCacheController {
         };
         this.cacheByHash.set(cacheKey, handle);
         this.updateLastCache(handle, elementHashes, params);
-        this.addToIndex(params, cacheKey);
+        this.addToIndex(params, cacheKey, effectiveCachePath);
         if (supersededRef) {
             this.release(supersededRef);
         }
@@ -581,7 +705,7 @@ export class MlxCacheController {
     }
     release(ref) {
         logger.debug('release', ref);
-        const entry = this.cacheIndex.entries.find(e => this.generateCachePath(e.key) === ref);
+        const entry = this.cacheIndex.entries.find(e => this.getEntryCachePath(e) === ref);
         if (entry) {
             entry.hint = 'release';
         }
@@ -607,6 +731,7 @@ export class MlxCacheController {
         ]);
         this.inflightRequests.clear();
         this.cacheByHash.clear();
+        this.cacheTokenCounts.clear();
         this.clearLastCache();
         if (this.managedDir && this.cacheDir) {
             await rm(this.cacheDir, { recursive: true, force: true }).catch(() => { });
@@ -614,7 +739,7 @@ export class MlxCacheController {
         else {
             const released = this.cacheIndex.entries.filter(e => e.hint === 'release');
             await Promise.allSettled(released.flatMap(entry => {
-                const path = this.generateCachePath(entry.key);
+                const path = this.getEntryCachePath(entry);
                 return [unlink(path), unlink(path + '.meta.json')];
             }));
             this.cacheIndex.entries = this.cacheIndex.entries.filter(e => e.hint !== 'release');