@modular-prompt/driver 0.13.4 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/mlx-ml/mlx-driver.d.ts.map +1 -1
- package/dist/mlx-ml/mlx-driver.js +74 -29
- package/dist/mlx-ml/mlx-driver.js.map +1 -1
- package/dist/mlx-ml/process/index.d.ts +1 -0
- package/dist/mlx-ml/process/index.d.ts.map +1 -1
- package/dist/mlx-ml/process/index.js +5 -1
- package/dist/mlx-ml/process/index.js.map +1 -1
- package/dist/mlx-ml/process/process-communication.d.ts +8 -0
- package/dist/mlx-ml/process/process-communication.d.ts.map +1 -1
- package/dist/mlx-ml/process/process-communication.js +29 -0
- package/dist/mlx-ml/process/process-communication.js.map +1 -1
- package/dist/mlx-ml/process/queue.d.ts +2 -0
- package/dist/mlx-ml/process/queue.d.ts.map +1 -1
- package/dist/mlx-ml/process/queue.js +6 -0
- package/dist/mlx-ml/process/queue.js.map +1 -1
- package/dist/mlx-ml/tool-call-parser/tool-formatter.js +2 -2
- package/dist/mlx-ml/tool-call-parser/tool-formatter.js.map +1 -1
- package/dist/query-utils.d.ts +29 -0
- package/dist/query-utils.d.ts.map +1 -0
- package/dist/query-utils.js +61 -0
- package/dist/query-utils.js.map +1 -0
- package/dist/types.d.ts +9 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/skills/driver-usage/SKILL.md +56 -1
- package/src/mlx-ml/python/handlers/cancel.py +53 -0
- package/src/mlx-ml/python/handlers/chat.py +3 -0
- package/src/mlx-ml/python/handlers/completion.py +3 -0
- package/src/mlx-ml/python/pyproject.toml +1 -1
- package/src/mlx-ml/python/server.py +7 -0
- package/src/mlx-ml/python/uv.lock +89 -798
|
@@ -218,6 +218,12 @@ export class QueueManager {
|
|
|
218
218
|
this.isProcessing = false;
|
|
219
219
|
this.processNext(); // 次のリクエストを処理
|
|
220
220
|
}
|
|
221
|
+
cancelActiveRequest() {
|
|
222
|
+
if (!this.isProcessing) {
|
|
223
|
+
return;
|
|
224
|
+
}
|
|
225
|
+
this.callbacks.cancelActiveStream();
|
|
226
|
+
}
|
|
221
227
|
get length() {
|
|
222
228
|
return this.queue.length;
|
|
223
229
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue.js","sourceRoot":"","sources":["../../../src/mlx-ml/process/queue.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"queue.js","sourceRoot":"","sources":["../../../src/mlx-ml/process/queue.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AA6B3D,MAAM,OAAO,YAAY;IACf,KAAK,GAAgB,EAAE,CAAC;IACxB,YAAY,GAAG,KAAK,CAAC;IACrB,SAAS,CAAwB;IAEzC,YAAY,SAAgC;QAC1C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED,sBAAsB;QACpB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,OAAO,GAA2B,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC;YACnE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;gBACd,OAAO;gBACP,OAAO;gBACP,MAAM;gBACN,kBAAkB,EAAE,IAAI;aACA,CAAC,CAAC;YAC5B,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,oBAAoB,CAAC,QAAsB,EAAE,OAA6B;QACxE,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,OAAO,GAAyB;gBACpC,MAAM,EAAE,aAAa;gBACrB,QAAQ;gBACR,OAAO;aACR,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;gBACd,OAAO;gBACP,OAAO;gBACP,MAAM;gBACN,kBAAkB,EAAE,IAAI;aACF,CAAC,CAAC;YAC1B,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,kBAAkB,CAAC,QAAsB,EAAE,KAA2B,EAAE,eAA2C;QACjH,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,OAAO,GAAuB;gBAClC,MAAM,EAAE,UAAU;gBAClB,QAAQ;gBACR,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;gBACvB,GAAG,CAAC,eAAe,IAAI,EAAE,gBAAgB,EAAE,eAAe,EAAE,CAAC;aAC9D,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;gBACd,OAAO;gBACP,OAAO;gBACP,MAAM;gBACN,kBAAkB,EAAE,IAAI;aACJ,CAAC,CAAC;YACxB,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,cAAc,CAAC,QAAsB,EAAE,MAAe,EAAE,OAA2B,EAAE,KAA2B,EAAE,MAAiB,EAAE,YAAqB,EAAE,eAA2C,EAAE,SAAkB,EAAE,eAAwB;QACnP,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,CAAC;gBACH,MAAM,OAAO,GAAmB;oBAC9B,MAAM,EAAE,MAAM;oBACd,QAAQ;oBACR,MAAM;oBACN,KAAK;oBACL,OAAO,EAAE,kBAAkB,CAAC,OAAO,EAAE,IAAI,CAAC;oBAC1C,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACnD,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACjE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC/C,GAAG,CAAC,eAAe,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAC3E,CAAC;gBACF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;oBACd,OAAO;oBACP,OAAO;oBACP,MAAM;iBACe,CAAC,CAAC;gBACzB,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,CAAC;YAChB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,sBAAsB,CAAC,SAAiB,EAAE,QAAsB,EAAE,aAAsB,EAAE,YAAqB,EAAE,aAAwB,EAAE,YAAuB,EAAE,KAA2B,EAAE,eAA2C;QAC1O,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,OAAO,GAA2B;gBACtC,MAAM,EAAE,eAAe;gBACvB,UAAU,EAAE,SAAS;gBACrB,QAAQ;gBACR,GAAG,CAAC,aAAa,IAAI,EAAE,eAAe,EAAE,aAAa,EAAE,CAAC;gBACxD,GAAG,CAAC,YAAY,IAAI,IAAI,IAAI,EAAE,cAAc,EAAE,YAAY,EAAE,CAAC;gBAC7D,GAAG,CAAC,aAAa,IAAI,YAAY,IAAI,EAAE,cAAc,EAAE,aAAa,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC;gBACpG,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;gBACvB,GAAG,CAAC,eAAe,IAAI,EAAE,gBAAgB,EAAE,eAAe,EAAE,CAAC;aAC9D,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;gBACd,OAAO;gBACP,OAAO;gBACP,MAAM;gBACN,kBAAkB,EAAE,IAAI;aACA,CAAC,CAAC;YAC5B,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,oBAAoB,CAAC,MAAc,EAAE,OAA2B,EAAE,MAAiB,EAAE,YAAqB;QACxG,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,CAAC;gBACH,MAAM,OAAO,GAAyB;oBACpC,MAAM,EAAE,YAAY;oBACpB,MAAM;oBACN,OAAO,EAAE,kBAAkB,CAAC,OAAO,EAAE,IAAI,CAAC;oBAC1C,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACpD,CAAC;gBACF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;oBACd,OAAO;oBACP,OAAO;oBACP,MAAM;iBACe,CAAC,CAAC;gBACzB,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,CAAC;YAChB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAGD,WAAW;QACT,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjD,OAAO;QACT,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa;QAC9C,MAAM,EAAE,OAAO,EAAE,kBAAkB,EAAE,GAAG,SAAS,CAAC;QAElD,oCAAoC;QACpC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACxB,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,eAAe,EAAE,CAAC;YAChD,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,aAAa;QACnC,CAAC;QAED,WAAW;QACX,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;IAC7C,CAAC;IAED,kBAAkB,CAAC,QAAgB;QACjC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACrC,IAAI,SAAS,EAAE,kBAAkB,EAAE,CAAC;gBAClC,IAAI,CAAC;oBACH,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;oBAC1C,uFAAuF;oBACvF,6FAA6F;oBAC7F,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,aAAa,EAAE,CAAC;wBAC/C,IAAI,kBAAkB,IAAI,YAAY,EAAE,CAAC;4BACtC,SAAiC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;wBAC3D,CAAC;6BAAM,CAAC;4BACL,SAAiC,CAAC,OAAO,CAAC;gCACzC,gBAAgB,EAAE,IAAI;gCACtB,gBAAgB,EAAE,KAAK;gCACvB,yBAAyB,EAAE,IAAI;gCAC/B,KAAK,EAAE,YAAY,CAAC,KAAK,IAAI,gCAAgC;6BAC9D,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;yBAAM,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;wBACnD,IAAI,aAAa,IAAI,YAAY,EAAE,CAAC;4BACjC,SAA+B,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;wBACzD,CAAC;6BAAM,CAAC;4BACL,SAA+B,CAAC,OAAO,CAAC;gCACvC,SAAS,EAAE,IAAI;gCACf,WAAW,EAAE,CAAC;gCACd,KAAK,EAAE,YAAY,CAAC,KAAK,IAAI,6BAA6B;6BAC3D,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;yBAAM,IAAI,YAAY,CAAC,KAAK,EAAE,CAAC;wBAC9B,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC;oBACpD,CAAC;yBAAM,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;wBACtD,SAAmC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;oBAC7D,CAAC;yBAAM,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,eAAe,EAAE,CAAC;wBACvD,SAAmC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;oBAC7D,CAAC;gBACH,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;wBAC/C,SAAmC,CAAC,OAAO,CAAC;4BAC3C,OAAO,EAAE,EAAE;4BACX,cAAc,EAAE,EAAE;4BAClB,QAAQ,EAAE,EAAE,mBAAmB,EAAE,KAAK,EAAE;yBACzC,CAAC,CAAC;oBACL,CAAC;yBAAM,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,aAAa,EAAE,CAAC;wBACrD,SAAiC,CAAC,OAAO,CAAC;4BACzC,gBAAgB,EAAE,IAAI;4BACtB,gBAAgB,EAAE,KAAK;4BACvB,yBAAyB,EAAE,IAAI;4BAC/B,KAAK,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;yBACxD,CAAC,CAAC;oBACL,CAAC;yBAAM,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;wBAClD,SAA+B,CAAC,OAAO,CAAC;4BACvC,SAAS,EAAE,IAAI;4BACf,WAAW,EAAE,CAAC;4BACd,KAAK,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;yBACxD,CAAC,CAAC;oBACL,CAAC;yBAAM,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,KAAK,eAAe,EAAE,CAAC;wBACvD,SAAmC,CAAC,MAAM,CACzC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAC9C,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,kBAAkB;QAChB,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,aAAa;IACnC,CAAC;IAED,mBAAmB;QACjB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE,CAAC;IACtC,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC;IACjC,CAAC;IAED,SAAS,CAAC,KAAY;QACpB,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACrC,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC3B,IAAI,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;QACD,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;IAC5B,CAAC;IAED,KAAK;QACH,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;IAC5B,CAAC;CACF"}
|
|
@@ -59,8 +59,8 @@ export function formatToolDefinitionsAsText(tools, specialTokens, toolCallFormat
|
|
|
59
59
|
lines.push('');
|
|
60
60
|
}
|
|
61
61
|
const concreteExample = exampleToolName
|
|
62
|
-
? `{"name": "${exampleToolName}", "arguments": ${exampleArgs}}`
|
|
63
|
-
: '{"name": "tool_name", "arguments": {"key": "value"}}';
|
|
62
|
+
? `{ "name": "${exampleToolName}", "arguments": ${exampleArgs} }`
|
|
63
|
+
: '{ "name": "tool_name", "arguments": {"key": "value"} }';
|
|
64
64
|
if (toolCallFormat?.call_start && toolCallFormat?.call_end) {
|
|
65
65
|
lines.push('To call a tool, respond ONLY with:');
|
|
66
66
|
lines.push(toolCallFormat.call_start);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tool-formatter.js","sourceRoot":"","sources":["../../../src/mlx-ml/tool-call-parser/tool-formatter.ts"],"names":[],"mappings":"AAGA,SAAS,gBAAgB,CACvB,KAAe,EACf,UAAmC,EACnC,QAAmB,EACnB,QAAgB,CAAC;IAEjB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAClC,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,MAAiC,CAAC;QACpD,MAAM,GAAG,GAAG,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,IAAI,KAAK,CAAC;QAErC,IAAI,IAAI,KAAK,OAAO,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;YACxC,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,KAAK,IAAI,UAAU,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC;YACrD,MAAM,KAAK,GAAG,SAAS,CAAC,KAAgC,CAAC;YACzD,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;gBACrB,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,cAAc,CAAC,CAAC;gBACpC,gBAAgB,CAAC,KAAK,EAAE,KAAK,CAAC,UAAqC,EAAE,KAAK,CAAC,QAAoB,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YAC9G,CAAC;QACH,CAAC;aAAM,IAAI,IAAI,KAAK,QAAQ,IAAI,SAAS,CAAC,UAAU,EAAE,CAAC;YACrD,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,KAAK,IAAI,WAAW,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC;YACtD,gBAAgB,CAAC,KAAK,EAAE,SAAS,CAAC,UAAqC,EAAE,SAAS,CAAC,QAAoB,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;QACtH,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,KAAK,IAAI,KAAK,IAAI,GAAG,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,2BAA2B,CACzC,KAAuB,EACvB,aAA+D,EAC/D,cAAsF;IAEtF,MAAM,KAAK,GAAa,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC;IACnD,KAAK,CAAC,IAAI,CAAC,6EAA6E,CAAC,CAAC;IAC1F,KAAK,CAAC,IAAI,CAAC,uFAAuF,CAAC,CAAC;IACpG,KAAK,CAAC,IAAI,CAAC,qIAAqI,CAAC,CAAC;IAClJ,KAAK,CAAC,IAAI,CAAC,2EAA2E,CAAC,CAAC;IACxF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,IAAI,eAAmC,CAAC;IACxC,IAAI,WAA+B,CAAC;IAEpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC/B,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC/B,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,MAAM,MAAM,GAAG,IAAI,CAAC,UAGnB,CAAC;YACF,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;gBACtB,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAC1B,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;gBAE/D,IAAI,CAAC,eAAe,EAAE,CAAC;oBACrB,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC;oBAC5B,MAAM,UAAU,GAAa,EAAE,CAAC;oBAChC,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;wBAC/D,MAAM,SAAS,GAAG,MAA2B,CAAC;wBAC9C,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;wBAChG,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,MAAM,GAAG,EAAE,CAAC,CAAC;oBACvC,CAAC;oBACD,WAAW,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;gBAC7C,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAC/D,CAAC;QACH,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,eAAe,GAAG,eAAe;QACrC,CAAC,CAAC,
|
|
1
|
+
{"version":3,"file":"tool-formatter.js","sourceRoot":"","sources":["../../../src/mlx-ml/tool-call-parser/tool-formatter.ts"],"names":[],"mappings":"AAGA,SAAS,gBAAgB,CACvB,KAAe,EACf,UAAmC,EACnC,QAAmB,EACnB,QAAgB,CAAC;IAEjB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAClC,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,MAAiC,CAAC;QACpD,MAAM,GAAG,GAAG,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,IAAI,KAAK,CAAC;QAErC,IAAI,IAAI,KAAK,OAAO,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;YACxC,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,KAAK,IAAI,UAAU,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC;YACrD,MAAM,KAAK,GAAG,SAAS,CAAC,KAAgC,CAAC;YACzD,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;gBACrB,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,cAAc,CAAC,CAAC;gBACpC,gBAAgB,CAAC,KAAK,EAAE,KAAK,CAAC,UAAqC,EAAE,KAAK,CAAC,QAAoB,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YAC9G,CAAC;QACH,CAAC;aAAM,IAAI,IAAI,KAAK,QAAQ,IAAI,SAAS,CAAC,UAAU,EAAE,CAAC;YACrD,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,KAAK,IAAI,WAAW,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC;YACtD,gBAAgB,CAAC,KAAK,EAAE,SAAS,CAAC,UAAqC,EAAE,SAAS,CAAC,QAAoB,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;QACtH,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,KAAK,IAAI,KAAK,IAAI,GAAG,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,2BAA2B,CACzC,KAAuB,EACvB,aAA+D,EAC/D,cAAsF;IAEtF,MAAM,KAAK,GAAa,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC;IACnD,KAAK,CAAC,IAAI,CAAC,6EAA6E,CAAC,CAAC;IAC1F,KAAK,CAAC,IAAI,CAAC,uFAAuF,CAAC,CAAC;IACpG,KAAK,CAAC,IAAI,CAAC,qIAAqI,CAAC,CAAC;IAClJ,KAAK,CAAC,IAAI,CAAC,2EAA2E,CAAC,CAAC;IACxF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,IAAI,eAAmC,CAAC;IACxC,IAAI,WAA+B,CAAC;IAEpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC/B,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC/B,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,MAAM,MAAM,GAAG,IAAI,CAAC,UAGnB,CAAC;YACF,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;gBACtB,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAC1B,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;gBAE/D,IAAI,CAAC,eAAe,EAAE,CAAC;oBACrB,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC;oBAC5B,MAAM,UAAU,GAAa,EAAE,CAAC;oBAChC,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;wBAC/D,MAAM,SAAS,GAAG,MAA2B,CAAC;wBAC9C,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;wBAChG,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,MAAM,GAAG,EAAE,CAAC,CAAC;oBACvC,CAAC;oBACD,WAAW,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;gBAC7C,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAC/D,CAAC;QACH,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,eAAe,GAAG,eAAe;QACrC,CAAC,CAAC,cAAc,eAAe,mBAAmB,WAAW,IAAI;QACjE,CAAC,CAAC,wDAAwD,CAAC;IAE7D,IAAI,cAAc,EAAE,UAAU,IAAI,cAAc,EAAE,QAAQ,EAAE,CAAC;QAC3D,KAAK,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;QACjD,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IACtC,CAAC;SAAM,CAAC;QACN,MAAM,aAAa,GAAG,aAAa,EAAE,SAAS,CAAC;QAC/C,IAAI,aAAa,IAAI,OAAO,IAAI,aAAa,IAAI,KAAK,IAAI,aAAa,EAAE,CAAC;YACxE,MAAM,IAAI,GAAG,aAAiC,CAAC;YAC/C,KAAK,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;YACjD,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;YACjC,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAC5B,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC,4HAA4H,CAAC,CAAC;YACzI,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YAC/B,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAC5B,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { QueryResult, StreamResult } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Token usage fields for {@link QueryResult.usage}.
|
|
4
|
+
* Drivers map provider-specific stats into this shape.
|
|
5
|
+
*/
|
|
6
|
+
export interface UsageCounts {
|
|
7
|
+
promptTokens?: number;
|
|
8
|
+
completionTokens?: number;
|
|
9
|
+
cacheReadTokens?: number;
|
|
10
|
+
cacheWriteTokens?: number;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Build {@link QueryResult.usage} from raw token counts.
|
|
14
|
+
* Omits the field when all counts are zero/undefined.
|
|
15
|
+
*/
|
|
16
|
+
export declare function buildQueryUsage(counts: UsageCounts): QueryResult['usage'] | undefined;
|
|
17
|
+
/** Whether an {@link AbortSignal} is already aborted. */
|
|
18
|
+
export declare function isAborted(signal?: AbortSignal): boolean;
|
|
19
|
+
/**
|
|
20
|
+
* Register an abort listener. Invokes `onAbort` immediately when already aborted.
|
|
21
|
+
* Returns a cleanup function.
|
|
22
|
+
*/
|
|
23
|
+
export declare function watchAbortSignal(signal: AbortSignal | undefined, onAbort: () => void): () => void;
|
|
24
|
+
/**
|
|
25
|
+
* StreamResult for a request that was aborted before inference started.
|
|
26
|
+
* `result` resolves (does not reject) with `finishReason: 'error'`.
|
|
27
|
+
*/
|
|
28
|
+
export declare function createAbortedStreamResult(extras?: Partial<QueryResult>): StreamResult;
|
|
29
|
+
//# sourceMappingURL=query-utils.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"query-utils.d.ts","sourceRoot":"","sources":["../src/query-utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE5D;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,WAAW,GAAG,WAAW,CAAC,OAAO,CAAC,GAAG,SAAS,CAsBrF;AAED,yDAAyD;AACzD,wBAAgB,SAAS,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAEvD;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,WAAW,GAAG,SAAS,EAC/B,OAAO,EAAE,MAAM,IAAI,GAClB,MAAM,IAAI,CAWZ;AAED;;;GAGG;AACH,wBAAgB,yBAAyB,CACvC,MAAM,GAAE,OAAO,CAAC,WAAW,CAAM,GAChC,YAAY,CAYd"}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Build {@link QueryResult.usage} from raw token counts.
|
|
3
|
+
* Omits the field when all counts are zero/undefined.
|
|
4
|
+
*/
|
|
5
|
+
export function buildQueryUsage(counts) {
|
|
6
|
+
const promptTokens = counts.promptTokens ?? 0;
|
|
7
|
+
const completionTokens = counts.completionTokens ?? 0;
|
|
8
|
+
const cacheReadTokens = counts.cacheReadTokens ?? 0;
|
|
9
|
+
const cacheWriteTokens = counts.cacheWriteTokens ?? 0;
|
|
10
|
+
if (promptTokens === 0 &&
|
|
11
|
+
completionTokens === 0 &&
|
|
12
|
+
cacheReadTokens === 0 &&
|
|
13
|
+
cacheWriteTokens === 0) {
|
|
14
|
+
return undefined;
|
|
15
|
+
}
|
|
16
|
+
return {
|
|
17
|
+
promptTokens,
|
|
18
|
+
completionTokens,
|
|
19
|
+
totalTokens: promptTokens + completionTokens,
|
|
20
|
+
...(cacheReadTokens > 0 ? { cacheReadTokens } : {}),
|
|
21
|
+
...(cacheWriteTokens > 0 ? { cacheWriteTokens } : {}),
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
/** Whether an {@link AbortSignal} is already aborted. */
|
|
25
|
+
export function isAborted(signal) {
|
|
26
|
+
return signal?.aborted ?? false;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Register an abort listener. Invokes `onAbort` immediately when already aborted.
|
|
30
|
+
* Returns a cleanup function.
|
|
31
|
+
*/
|
|
32
|
+
export function watchAbortSignal(signal, onAbort) {
|
|
33
|
+
if (!signal) {
|
|
34
|
+
return () => { };
|
|
35
|
+
}
|
|
36
|
+
if (signal.aborted) {
|
|
37
|
+
onAbort();
|
|
38
|
+
return () => { };
|
|
39
|
+
}
|
|
40
|
+
const handler = () => onAbort();
|
|
41
|
+
signal.addEventListener('abort', handler);
|
|
42
|
+
return () => signal.removeEventListener('abort', handler);
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* StreamResult for a request that was aborted before inference started.
|
|
46
|
+
* `result` resolves (does not reject) with `finishReason: 'error'`.
|
|
47
|
+
*/
|
|
48
|
+
export function createAbortedStreamResult(extras = {}) {
|
|
49
|
+
const emptyStream = {
|
|
50
|
+
async *[Symbol.asyncIterator]() { },
|
|
51
|
+
};
|
|
52
|
+
return {
|
|
53
|
+
stream: emptyStream,
|
|
54
|
+
result: Promise.resolve({
|
|
55
|
+
content: '',
|
|
56
|
+
finishReason: 'error',
|
|
57
|
+
...extras,
|
|
58
|
+
}),
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
//# sourceMappingURL=query-utils.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"query-utils.js","sourceRoot":"","sources":["../src/query-utils.ts"],"names":[],"mappings":"AAaA;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,MAAmB;IACjD,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,CAAC,CAAC;IAC9C,MAAM,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,CAAC,CAAC;IACtD,MAAM,eAAe,GAAG,MAAM,CAAC,eAAe,IAAI,CAAC,CAAC;IACpD,MAAM,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,CAAC,CAAC;IAEtD,IACE,YAAY,KAAK,CAAC;QAClB,gBAAgB,KAAK,CAAC;QACtB,eAAe,KAAK,CAAC;QACrB,gBAAgB,KAAK,CAAC,EACtB,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO;QACL,YAAY;QACZ,gBAAgB;QAChB,WAAW,EAAE,YAAY,GAAG,gBAAgB;QAC5C,GAAG,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnD,GAAG,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtD,CAAC;AACJ,CAAC;AAED,yDAAyD;AACzD,MAAM,UAAU,SAAS,CAAC,MAAoB;IAC5C,OAAO,MAAM,EAAE,OAAO,IAAI,KAAK,CAAC;AAClC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAC9B,MAA+B,EAC/B,OAAmB;IAEnB,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,GAAG,EAAE,GAAE,CAAC,CAAC;IAClB,CAAC;IACD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,OAAO,EAAE,CAAC;QACV,OAAO,GAAG,EAAE,GAAE,CAAC,CAAC;IAClB,CAAC;IACD,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;IAChC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC1C,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;AAC5D,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,yBAAyB,CACvC,SAA+B,EAAE;IAEjC,MAAM,WAAW,GAA0B;QACzC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,KAAI,CAAC;KACnC,CAAC;IACF,OAAO;QACL,MAAM,EAAE,WAAW;QACnB,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC;YACtB,OAAO,EAAE,EAAE;YACX,YAAY,EAAE,OAAO;YACrB,GAAG,MAAM;SACV,CAAC;KACH,CAAC;AACJ,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -88,6 +88,10 @@ export interface QueryResult {
|
|
|
88
88
|
promptTokens: number;
|
|
89
89
|
completionTokens: number;
|
|
90
90
|
totalTokens: number;
|
|
91
|
+
/** Tokens read from prompt/KV cache in this request */
|
|
92
|
+
cacheReadTokens?: number;
|
|
93
|
+
/** Tokens newly written to prompt/KV cache in this request */
|
|
94
|
+
cacheWriteTokens?: number;
|
|
91
95
|
};
|
|
92
96
|
/** Tool calls selected by the model */
|
|
93
97
|
toolCalls?: ToolCall[];
|
|
@@ -141,6 +145,11 @@ export interface QueryOptions {
|
|
|
141
145
|
* - undefined: driver default behavior
|
|
142
146
|
*/
|
|
143
147
|
cache?: boolean | 'read-only';
|
|
148
|
+
/**
|
|
149
|
+
* Abort signal for cancelling in-flight inference.
|
|
150
|
+
* Unsupported drivers ignore this option.
|
|
151
|
+
*/
|
|
152
|
+
signal?: AbortSignal;
|
|
144
153
|
}
|
|
145
154
|
/**
|
|
146
155
|
* Stream result with both stream and final result
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACjF,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAGtD,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAEjG;;GAEG;AACH,MAAM,MAAM,IAAI,GAAG,QAAQ,GAAG,WAAW,GAAG,MAAM,GAAG,MAAM,CAAC;AAE5D;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;IACtC,OAAO,EAAE,MAAM,GAAG,UAAU,EAAE,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,IAAI,EAAE,WAAW,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,QAAQ,EAAE,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,6CAA6C;IAC7C,UAAU,EAAE,MAAM,CAAC;IACnB,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,+BAA+B;IAC/B,IAAI,EAAE,cAAc,CAAC;IACrB,8BAA8B;IAC9B,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,mBAAmB,GAAG,wBAAwB,GAAG,iBAAiB,CAAC;AAE7F;;GAEG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,IAAI,wBAAwB,CAEtF;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,IAAI,iBAAiB,CAE/E;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,yDAAyD;IACzD,IAAI,EAAE,MAAM,CAAC;IACb,oEAAoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wCAAwC;IACxC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,0DAA0D;IAC1D,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,MAAM,UAAU,GAClB,MAAM,GACN,MAAM,GACN,UAAU,GACV;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAGrB;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAE3B,KAAK,CAAC,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,gBAAgB,EAAE,MAAM,CAAC;QACzB,WAAW,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACjF,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAGtD,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAEjG;;GAEG;AACH,MAAM,MAAM,IAAI,GAAG,QAAQ,GAAG,WAAW,GAAG,MAAM,GAAG,MAAM,CAAC;AAE5D;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;IACtC,OAAO,EAAE,MAAM,GAAG,UAAU,EAAE,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,IAAI,EAAE,WAAW,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,QAAQ,EAAE,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,6CAA6C;IAC7C,UAAU,EAAE,MAAM,CAAC;IACnB,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,+BAA+B;IAC/B,IAAI,EAAE,cAAc,CAAC;IACrB,8BAA8B;IAC9B,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,mBAAmB,GAAG,wBAAwB,GAAG,iBAAiB,CAAC;AAE7F;;GAEG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,IAAI,wBAAwB,CAEtF;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,IAAI,iBAAiB,CAE/E;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,yDAAyD;IACzD,IAAI,EAAE,MAAM,CAAC;IACb,oEAAoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wCAAwC;IACxC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,0DAA0D;IAC1D,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,MAAM,UAAU,GAClB,MAAM,GACN,MAAM,GACN,UAAU,GACV;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAGrB;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAE3B,KAAK,CAAC,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,gBAAgB,EAAE,MAAM,CAAC;QACzB,WAAW,EAAE,MAAM,CAAC;QACpB,uDAAuD;QACvD,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,8DAA8D;QAC9D,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC3B,CAAC;IAEF,uCAAuC;IACvC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAEvB,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B,kDAAkD;IAClD,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;IAExB,8DAA8D;IAC9D,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,GAAG,YAAY,CAAC;AAEtE;;;;;;GAMG;AACH,MAAM,MAAM,SAAS,GAAG,SAAS,GAAG,UAAU,GAAG,UAAU,GAAG,MAAM,CAAC;AAErE;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,8EAA8E;IAC9E,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,YAAY,GAAG,kBAAkB,CAAC;IACzD,iCAAiC;IACjC,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IACzB,0BAA0B;IAC1B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,qFAAqF;IACrF,eAAe,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IAC5C;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,OAAO,GAAG,WAAW,CAAC;IAC9B;;;OAGG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B;;OAEG;IACH,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;IAE9B;;OAEG;IACH,MAAM,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB;;OAEG;IACH,KAAK,CAAC,MAAM,EAAE,OAAO,sBAAsB,EAAE,cAAc,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAE3G;;OAEG;IACH,WAAW,CAAC,MAAM,EAAE,OAAO,sBAAsB,EAAE,cAAc,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAElH;;OAEG;IACH,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,QAAQ,GAAG,WAAW,GAAG,QAAQ,GAAG,MAAM,CAAC;IACrD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,YAAY,CAAC;CAC/B"}
|
package/package.json
CHANGED
|
@@ -54,6 +54,8 @@ interface QueryOptions {
|
|
|
54
54
|
stream?: boolean;
|
|
55
55
|
tools?: ToolDefinition[];
|
|
56
56
|
toolChoice?: ToolChoice;
|
|
57
|
+
cache?: boolean | 'read-only'; // プロンプトキャッシュ（ドライバー依存）
|
|
58
|
+
signal?: AbortSignal; // 推論キャンセル（未対応ドライバーは無視）
|
|
57
59
|
}
|
|
58
60
|
```
|
|
59
61
|
|
|
@@ -67,6 +69,8 @@ interface QueryResult {
|
|
|
67
69
|
promptTokens: number;
|
|
68
70
|
completionTokens: number;
|
|
69
71
|
totalTokens: number;
|
|
72
|
+
cacheReadTokens?: number; // キャッシュヒット分（MLX 等）
|
|
73
|
+
cacheWriteTokens?: number; // 新規キャッシュ書き込み分（MLX 等）
|
|
70
74
|
};
|
|
71
75
|
toolCalls?: ToolCall[]; // ツール呼び出し
|
|
72
76
|
finishReason?: FinishReason; // 'stop' | 'length' | 'error' | 'tool_calls'
|
|
@@ -78,10 +82,47 @@ interface QueryResult {
|
|
|
78
82
|
```typescript
|
|
79
83
|
interface StreamResult {
|
|
80
84
|
stream: AsyncIterable<string>; // テキストチャンクのストリーム
|
|
81
|
-
result: Promise<QueryResult>; //
|
|
85
|
+
result: Promise<QueryResult>; // 最終結果（ストリーム完了または abort 後に解決）
|
|
82
86
|
}
|
|
83
87
|
```
|
|
84
88
|
|
|
89
|
+
- usage は `stream` チャンクではなく **`result.usage` のみ**
|
|
90
|
+
- `signal` でキャンセルした場合も `result` は reject しない（`finishReason: 'error'` + `signal.aborted` で判定）
|
|
91
|
+
|
|
92
|
+
## 推論キャンセル（AbortSignal）
|
|
93
|
+
|
|
94
|
+
```typescript
|
|
95
|
+
const controller = new AbortController();
|
|
96
|
+
const { stream, result } = await driver.streamQuery(compiled, {
|
|
97
|
+
signal: controller.signal,
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
for await (const chunk of stream) {
|
|
101
|
+
process.stdout.write(chunk);
|
|
102
|
+
if (userPressedEsc) controller.abort();
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const final = await result;
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
| ドライバー | 対応 |
|
|
109
|
+
|---|---|
|
|
110
|
+
| MlxDriver | ✅ stdin cancel + Python `poll_cancel()` |
|
|
111
|
+
| その他 | 未実装（`signal` 無視） |
|
|
112
|
+
|
|
113
|
+
ヘルパー: `watchAbortSignal`, `createAbortedStreamResult`, `isAborted`（`@modular-prompt/driver`）
|
|
114
|
+
|
|
115
|
+
## トークン使用量とキャッシュ
|
|
116
|
+
|
|
117
|
+
```typescript
|
|
118
|
+
const final = await result;
|
|
119
|
+
final.usage?.promptTokens; // プロバイダ報告の prompt 側総数
|
|
120
|
+
final.usage?.cacheReadTokens; // MLX: KV キャッシュヒット分
|
|
121
|
+
final.usage?.cacheWriteTokens; // MLX: 同一クエリ内の新規 prefill 分
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
`buildQueryUsage()` でカスタムドライバーから同形式の usage を組み立てられます。
|
|
125
|
+
|
|
85
126
|
## 各ドライバーのConfig
|
|
86
127
|
|
|
87
128
|
### OpenAIDriver
|
|
@@ -219,6 +260,20 @@ await driver.close();
|
|
|
219
260
|
|
|
220
261
|
Apple Silicon専用。Python 3.11以上が必要。
|
|
221
262
|
|
|
263
|
+
#### AbortSignal（推論キャンセル）
|
|
264
|
+
|
|
265
|
+
`QueryOptions.signal` を渡すと、ストリーム中の推論をキャンセルできます。
|
|
266
|
+
|
|
267
|
+
```typescript
|
|
268
|
+
const controller = new AbortController();
|
|
269
|
+
const { stream, result } = await driver.streamQuery(compiled, {
|
|
270
|
+
signal: controller.signal,
|
|
271
|
+
});
|
|
272
|
+
// controller.abort() → Python 子プロセスに cancel 送信、部分 content 保持
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
実装: TS が stdin に `{"method":"cancel"}` を送り、Python の `stream_generate` ループが `poll_cancel()` で協調的に終了します。
|
|
276
|
+
|
|
222
277
|
#### textOnlyオプション
|
|
223
278
|
|
|
224
279
|
VLM（Vision Language Model）対応モデルを画像なしのテキストのみで使用する場合に`textOnly: true`を指定します。
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Cancel request handling for in-flight streaming generation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import select
|
|
7
|
+
import sys
|
|
8
|
+
|
|
9
|
+
_cancel_requested = False
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def request_cancel() -> None:
|
|
13
|
+
global _cancel_requested
|
|
14
|
+
_cancel_requested = True
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def reset_cancel() -> None:
|
|
18
|
+
global _cancel_requested
|
|
19
|
+
_cancel_requested = False
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def is_cancel_requested() -> bool:
|
|
23
|
+
return _cancel_requested
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def poll_cancel() -> bool:
|
|
27
|
+
"""Non-blocking check for a cancel command on stdin during streaming."""
|
|
28
|
+
global _cancel_requested
|
|
29
|
+
if _cancel_requested:
|
|
30
|
+
return True
|
|
31
|
+
|
|
32
|
+
try:
|
|
33
|
+
ready, _, _ = select.select([sys.stdin], [], [], 0)
|
|
34
|
+
except (ValueError, OSError):
|
|
35
|
+
return False
|
|
36
|
+
|
|
37
|
+
if not ready:
|
|
38
|
+
return False
|
|
39
|
+
|
|
40
|
+
line = sys.stdin.readline()
|
|
41
|
+
if not line:
|
|
42
|
+
return False
|
|
43
|
+
|
|
44
|
+
try:
|
|
45
|
+
req = json.loads(line)
|
|
46
|
+
except json.JSONDecodeError:
|
|
47
|
+
return False
|
|
48
|
+
|
|
49
|
+
if req.get("method") == "cancel":
|
|
50
|
+
_cancel_requested = True
|
|
51
|
+
return True
|
|
52
|
+
|
|
53
|
+
return False
|
|
@@ -7,6 +7,7 @@ import sys
|
|
|
7
7
|
from backends.base import ModelBackend
|
|
8
8
|
from mlx_lm.models.cache import trim_prompt_cache
|
|
9
9
|
from utils.prompt_builder import generate_merged_prompt, supports_chat_template
|
|
10
|
+
from handlers.cancel import poll_cancel
|
|
10
11
|
|
|
11
12
|
|
|
12
13
|
def _read_cache_token_count(cache_path: str) -> int | None:
|
|
@@ -34,6 +35,8 @@ def _stream_to_stdout(
|
|
|
34
35
|
|
|
35
36
|
last_response = None
|
|
36
37
|
for response in backend.stream_generate(prompt, options, images, prompt_cache=prompt_cache):
|
|
38
|
+
if poll_cancel():
|
|
39
|
+
break
|
|
37
40
|
print(response.text.replace("\0", "").replace("\x1e", ""), end="", flush=True)
|
|
38
41
|
last_response = response
|
|
39
42
|
|
|
@@ -5,6 +5,7 @@ import re
|
|
|
5
5
|
import sys
|
|
6
6
|
|
|
7
7
|
from backends.base import ModelBackend
|
|
8
|
+
from handlers.cancel import poll_cancel
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
def handle_completion(
|
|
@@ -31,6 +32,8 @@ def handle_completion(
|
|
|
31
32
|
sys.stderr.write(f"--- prompt\n{prompt}\n")
|
|
32
33
|
|
|
33
34
|
for response in backend.stream_generate(prompt, final_options, images):
|
|
35
|
+
if poll_cancel():
|
|
36
|
+
break
|
|
34
37
|
print(response.text.replace("\0", ""), end="", flush=True)
|
|
35
38
|
|
|
36
39
|
print("\n", end="\0", flush=True)
|
|
@@ -9,7 +9,7 @@ dependencies = [
|
|
|
9
9
|
"jinja2==3.1.6",
|
|
10
10
|
"mlx>=0.31.2; sys_platform == 'darwin'",
|
|
11
11
|
"mlx-lm==0.31.3; sys_platform == 'darwin'",
|
|
12
|
-
"mlx-vlm==0.
|
|
12
|
+
"mlx-vlm==0.6.3; sys_platform == 'darwin'",
|
|
13
13
|
"tokenizers==0.22.2",
|
|
14
14
|
"torch==2.9.1",
|
|
15
15
|
"torchvision==0.24.1",
|
|
@@ -4,6 +4,7 @@ import sys
|
|
|
4
4
|
|
|
5
5
|
from backends.base import ModelBackend
|
|
6
6
|
from handlers import handle_cache_prefill, handle_capabilities, handle_chat, handle_completion, handle_format_test, handle_tokenize
|
|
7
|
+
from handlers.cancel import request_cancel, reset_cancel
|
|
7
8
|
|
|
8
9
|
|
|
9
10
|
MAX_READ_LINES = 10000
|
|
@@ -48,6 +49,12 @@ class Server:
|
|
|
48
49
|
self._error_response("'method' field is required")
|
|
49
50
|
return
|
|
50
51
|
|
|
52
|
+
if method == 'cancel':
|
|
53
|
+
request_cancel()
|
|
54
|
+
return
|
|
55
|
+
|
|
56
|
+
reset_cancel()
|
|
57
|
+
|
|
51
58
|
try:
|
|
52
59
|
if method == 'capabilities':
|
|
53
60
|
handle_capabilities(self.capabilities)
|