ai.matey.types 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/cjs/adapters.js +16 -0
- package/dist/cjs/adapters.js.map +1 -0
- package/dist/cjs/bridge.js +32 -0
- package/dist/cjs/bridge.js.map +1 -0
- package/dist/cjs/errors.js +121 -0
- package/dist/cjs/errors.js.map +1 -0
- package/dist/cjs/index.js +44 -0
- package/dist/cjs/index.js.map +1 -0
- package/dist/cjs/ir.js +17 -0
- package/dist/cjs/ir.js.map +1 -0
- package/dist/cjs/middleware.js +11 -0
- package/dist/cjs/middleware.js.map +1 -0
- package/dist/cjs/model-runner.js +11 -0
- package/dist/cjs/model-runner.js.map +1 -0
- package/dist/cjs/model-translation.js +10 -0
- package/dist/cjs/model-translation.js.map +1 -0
- package/dist/cjs/models.js +11 -0
- package/dist/cjs/models.js.map +1 -0
- package/dist/cjs/router.js +93 -0
- package/dist/cjs/router.js.map +1 -0
- package/dist/cjs/streaming.js +28 -0
- package/dist/cjs/streaming.js.map +1 -0
- package/dist/esm/adapters.js +15 -0
- package/dist/esm/adapters.js.map +1 -0
- package/dist/esm/bridge.js +29 -0
- package/dist/esm/bridge.js.map +1 -0
- package/dist/esm/errors.js +118 -0
- package/dist/esm/errors.js.map +1 -0
- package/dist/esm/index.js +28 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/ir.js +16 -0
- package/dist/esm/ir.js.map +1 -0
- package/dist/esm/middleware.js +10 -0
- package/dist/esm/middleware.js.map +1 -0
- package/dist/esm/model-runner.js +10 -0
- package/dist/esm/model-runner.js.map +1 -0
- package/dist/esm/model-translation.js +9 -0
- package/dist/esm/model-translation.js.map +1 -0
- package/dist/esm/models.js +10 -0
- package/dist/esm/models.js.map +1 -0
- package/dist/esm/router.js +90 -0
- package/dist/esm/router.js.map +1 -0
- package/dist/esm/streaming.js +25 -0
- package/dist/esm/streaming.js.map +1 -0
- package/dist/types/adapters.d.ts +377 -0
- package/dist/types/adapters.d.ts.map +1 -0
- package/dist/types/bridge.d.ts +290 -0
- package/dist/types/bridge.d.ts.map +1 -0
- package/dist/types/errors.d.ts +380 -0
- package/dist/types/errors.d.ts.map +1 -0
- package/dist/types/index.d.ts +18 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/ir.d.ts +820 -0
- package/dist/types/ir.d.ts.map +1 -0
- package/dist/types/middleware.d.ts +256 -0
- package/dist/types/middleware.d.ts.map +1 -0
- package/dist/types/model-runner.d.ts +344 -0
- package/dist/types/model-runner.d.ts.map +1 -0
- package/dist/types/model-translation.d.ts +76 -0
- package/dist/types/model-translation.d.ts.map +1 -0
- package/dist/types/models.d.ts +160 -0
- package/dist/types/models.d.ts.map +1 -0
- package/dist/types/router.d.ts +526 -0
- package/dist/types/router.d.ts.map +1 -0
- package/dist/types/streaming.d.ts +96 -0
- package/dist/types/streaming.d.ts.map +1 -0
- package/package.json +64 -0
- package/readme.md +84 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ir.d.ts","sourceRoot":"","sources":["../../src/ir.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAMjD;;;;;;;;GAQG;AACH,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,MAAM,GAAG,WAAW,GAAG,MAAM,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,MAAM,EACX;QACE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;QACrB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;KACtB,GACD;QACE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;QACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;KACvB,CAAC;CACP;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,WAAW,EAAE,CAAC;IACzC,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED;;;;GAIG;AACH,MAAM,MAAM,cAAc,GAAG,WAAW,GAAG,YAAY,GAAG,cAAc,GAAG,iBAAiB,CAAC;AAE7F;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,MAAM,WAAW,SAAS;IACxB;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAE3B;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,cAAc,EAAE,CAAC;IAErD;;;OAGG;IACH,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC7C;AAMD;;GAEG;AACH,MAAM,MAAM,cAAc,GACtB,QAAQ,GACR,QAAQ,GACR,SAAS,GACT,SAAS,GACT,QAAQ,GACR,OAAO,GACP,MAAM,CAAC;AAEX;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,cAAc,GAAG,SAAS,cAAc,EAAE,CAAC;IAC3D,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,OAAO,EAAE,CAAC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACjD,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,QAAQ,CAAC,KAAK,CAAC,EAAE,UAAU,CAAC;IAC5B,QAAQ,CAAC,oBAAoB,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC;IACrD,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,OAAO,EAAE,CAAC;CACxC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,WAAW,MAAM;IACrB;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB;;OAEG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAE7B;;OAEG;IACH,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;IAEhC;;OAEG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC7C;AAMD;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,WAAW,YAAY;IAC3B;;;OAGG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAExB;;;;OAIG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAE9B;;;OAGG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAE5B;;;OAGG;IACH,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAEnC;;;OAGG;IACH,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAElC;;;OAGG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAE3C;;OAEG;IACH,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAEvB;;OAEG;IACH,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC3C;AAMD;;GAEG;AACH,MAAM,MAAM,qBAAqB,GAC7B,oBAAoB,GACpB,aAAa,GACb,cAAc,GACd,eAAe,CAAC;AAEpB;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,WAAW,cAAc;IAC7B;;OAEG;IACH,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAE5B;;OAEG;IACH,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;IAE7B;;OAEG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IAEzB;;OAEG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAEnC;;OAEG;IACH,QAAQ,CAAC,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAE7C;;OAEG;IACH,QAAQ,CAAC,qBAAqB,EAAE,qBAAqB,CAAC;IAEtD;;OAEG;IACH,QAAQ,CAAC,8BAA8B,EAAE,OAAO,CAAC;IAEjD;;OAEG;IACH,QAAQ,CAAC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAEvC;;OAEG;IACH,QAAQ,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC;IAEhC;;OAEG;IACH,QAAQ,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC;IAEhC;;OAEG;IACH,QAAQ,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC;IAEhC;;OAEG;IACH,QAAQ,CAAC,wBAAwB,CAAC,EAAE,OAAO,CAAC;IAE5C;;OAEG;IACH,QAAQ,CAAC,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAE3C;;OAEG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACpC;AAMD;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;AAE3D;;GAEG;AACH,MAAM,MAAM,eAAe,GACvB,sBAAsB,GACtB,mBAAmB,GACnB,uBAAuB,GACvB,wBAAwB,GACxB,sBAAsB,GACtB,0BAA0B,GAC1B,4BAA4B,GAC5B,0BAA0B,GAC1B,kBAAkB,GAClB,mBAAmB,CAAC;AAExB;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,WAAW,SAAS;IACxB;;OAEG;IACH,QAAQ,CAAC,QAAQ,EAAE,eAAe,CAAC;IAEnC;;OAEG;IACH,QAAQ,CAAC,QAAQ,EAAE,eAAe,CAAC;IAEnC;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAEzB;;OAEG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,CAAC;IAEjC;;OAEG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAEpC;;OAEG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAEzB;;OAEG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC5C;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B;;OAEG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAE3B;;OAEG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAE1B;;OAEG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAExC;;OAEG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,MAAM,WAAW,UAAU;IACzB;;;;OAIG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAE3B;;;;;OAKG;IACH,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAErC;;OAEG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAE3B;;OAEG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,YAAY,CAAC;IAEnC;;;OAGG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,SAAS,EAAE,CAAC;IAEzC;;;OAGG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC3C;AAMD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,WAAW,aAAa;IAC5B;;;OAGG;IACH,QAAQ,CAAC,QAAQ,EAAE,SAAS,SAAS,EAAE,CAAC;IAExC;;;OAGG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAEnC;;;;;;OAMG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,UAAU,GAAG,MAAM,GAAG;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAE/E;;OAEG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,YAAY,CAAC;IAEnC;;OAEG;IACH,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAE9B;;;OAGG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;;;;;;;;;;;OAcG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;CAClC;AAMD;;GAEG;AACH,MAAM,MAAM,YAAY,GACpB,MAAM,GACN,QAAQ,GACR,YAAY,GACZ,gBAAgB,GAChB,OAAO,GACP,WAAW,CAAC;AAEhB;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB;;OAEG;IACH,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAE9B;;OAEG;IACH,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAElC;;OAEG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAE7B;;OAEG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC5C;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,MAAM,WAAW,cAAc;IAC7B;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC;IAE5B;;OAEG;IACH,QAAQ,CAAC,YAAY,EAAE,YAAY,CAAC;IAEpC;;OAEG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IAEzB;;;OAGG;IACH,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAE9B;;OAEG;IACH,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACxC;AAMD;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,OAAO,GAAG,SAAS,GAAG,UAAU,GAAG,UAAU,GAAG,MAAM,GAAG,OAAO,CAAC;AAE/F;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,IAAI,EAAE,eAAe,CAAC;IAC/B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAiB,SAAQ,eAAe;IACvD,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;CAC/B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,WAAW,kBAAmB,SAAQ,eAAe;IACzD,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAEzB;;;;OAIG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAEvB;;;;OAIG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAE9B,QAAQ,CAAC,IAAI,CAAC,EAAE,WAAW,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,kBAAmB,SAAQ,eAAe;IACzD,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,mBAAoB,SAAQ,eAAe;IAC1D,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAClC,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,eAAgB,SAAQ,eAAe;IACtD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,YAAY,EAAE,YAAY,CAAC;IACpC,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAiB,SAAQ,eAAe;IACvD,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE;QACd,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KAC5C,CAAC;CACH;AAED;;;;GAIG;AACH,MAAM,MAAM,aAAa,GACrB,gBAAgB,GAChB,kBAAkB,GAClB,kBAAkB,GAClB,mBAAmB,GACnB,eAAe,GACf,gBAAgB,CAAC;AAErB;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,MAAM,YAAY,GAAG,cAAc,CAAC,aAAa,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC"}
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Middleware Function Signatures
|
|
3
|
+
*
|
|
4
|
+
* Middleware provides composable transformation layers for cross-cutting concerns
|
|
5
|
+
* like logging, caching, telemetry, prompt rewriting, and error handling.
|
|
6
|
+
*
|
|
7
|
+
* @module
|
|
8
|
+
*/
|
|
9
|
+
import type { IRChatRequest, IRChatResponse, IRChatStream, IRStreamChunk } from './ir.js';
|
|
10
|
+
import type { BackendAdapter } from './adapters.js';
|
|
11
|
+
/**
|
|
12
|
+
* Context passed to middleware during execution.
|
|
13
|
+
*/
|
|
14
|
+
export interface MiddlewareContext {
|
|
15
|
+
/**
|
|
16
|
+
* The IR request being processed.
|
|
17
|
+
* Middleware can inspect and modify this.
|
|
18
|
+
*/
|
|
19
|
+
request: IRChatRequest;
|
|
20
|
+
/**
|
|
21
|
+
* Whether this is a streaming request.
|
|
22
|
+
*/
|
|
23
|
+
readonly isStreaming: boolean;
|
|
24
|
+
/**
|
|
25
|
+
* Backend that will process (or processed) the request.
|
|
26
|
+
* Available after routing decision.
|
|
27
|
+
*/
|
|
28
|
+
readonly backend?: BackendAdapter;
|
|
29
|
+
/**
|
|
30
|
+
* Backend name/identifier.
|
|
31
|
+
*/
|
|
32
|
+
readonly backendName?: string;
|
|
33
|
+
/**
|
|
34
|
+
* Shared state object for passing data between middleware.
|
|
35
|
+
*/
|
|
36
|
+
readonly state: Record<string, unknown>;
|
|
37
|
+
/**
|
|
38
|
+
* Configuration from bridge.
|
|
39
|
+
*/
|
|
40
|
+
readonly config: Record<string, unknown>;
|
|
41
|
+
/**
|
|
42
|
+
* Abort signal for request cancellation.
|
|
43
|
+
*/
|
|
44
|
+
readonly signal?: AbortSignal;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Context for streaming middleware.
|
|
48
|
+
*/
|
|
49
|
+
export interface StreamingMiddlewareContext extends MiddlewareContext {
|
|
50
|
+
readonly isStreaming: true;
|
|
51
|
+
/**
|
|
52
|
+
* Current stream chunk being processed.
|
|
53
|
+
*/
|
|
54
|
+
chunk?: IRStreamChunk;
|
|
55
|
+
/**
|
|
56
|
+
* Total chunks processed so far.
|
|
57
|
+
*/
|
|
58
|
+
readonly chunksProcessed: number;
|
|
59
|
+
/**
|
|
60
|
+
* Whether stream has completed.
|
|
61
|
+
*/
|
|
62
|
+
readonly streamComplete: boolean;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Next function in middleware chain.
|
|
66
|
+
*/
|
|
67
|
+
export type MiddlewareNext = () => Promise<IRChatResponse>;
|
|
68
|
+
/**
|
|
69
|
+
* Next function for streaming middleware chain.
|
|
70
|
+
*/
|
|
71
|
+
export type StreamingMiddlewareNext = () => Promise<IRChatStream>;
|
|
72
|
+
/**
|
|
73
|
+
* Standard middleware function.
|
|
74
|
+
*/
|
|
75
|
+
export type Middleware = (context: MiddlewareContext, next: MiddlewareNext) => Promise<IRChatResponse>;
|
|
76
|
+
/**
|
|
77
|
+
* Streaming middleware function.
|
|
78
|
+
*/
|
|
79
|
+
export type StreamingMiddleware = (context: StreamingMiddlewareContext, next: StreamingMiddlewareNext) => Promise<IRChatStream>;
|
|
80
|
+
/**
|
|
81
|
+
* Options for creating middleware.
|
|
82
|
+
*/
|
|
83
|
+
export interface MiddlewareOptions {
|
|
84
|
+
/**
|
|
85
|
+
* Middleware name for debugging/logging.
|
|
86
|
+
*/
|
|
87
|
+
name?: string;
|
|
88
|
+
/**
|
|
89
|
+
* Whether middleware supports streaming.
|
|
90
|
+
* @default false
|
|
91
|
+
*/
|
|
92
|
+
supportsStreaming?: boolean;
|
|
93
|
+
/**
|
|
94
|
+
* Whether middleware should run before routing decision.
|
|
95
|
+
* @default false
|
|
96
|
+
*/
|
|
97
|
+
runBeforeRouting?: boolean;
|
|
98
|
+
/**
|
|
99
|
+
* Custom configuration for this middleware.
|
|
100
|
+
*/
|
|
101
|
+
config?: Record<string, unknown>;
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Middleware with metadata.
|
|
105
|
+
*/
|
|
106
|
+
export interface MiddlewareWithMetadata {
|
|
107
|
+
/**
|
|
108
|
+
* Middleware name.
|
|
109
|
+
*/
|
|
110
|
+
readonly name: string;
|
|
111
|
+
/**
|
|
112
|
+
* Whether middleware supports streaming.
|
|
113
|
+
*/
|
|
114
|
+
readonly supportsStreaming: boolean;
|
|
115
|
+
/**
|
|
116
|
+
* Whether middleware runs before routing.
|
|
117
|
+
*/
|
|
118
|
+
readonly runBeforeRouting: boolean;
|
|
119
|
+
/**
|
|
120
|
+
* Middleware configuration.
|
|
121
|
+
*/
|
|
122
|
+
readonly config: Record<string, unknown>;
|
|
123
|
+
/**
|
|
124
|
+
* The middleware function.
|
|
125
|
+
*/
|
|
126
|
+
readonly middleware: Middleware;
|
|
127
|
+
/**
|
|
128
|
+
* The streaming middleware function (if supported).
|
|
129
|
+
*/
|
|
130
|
+
readonly streamingMiddleware?: StreamingMiddleware;
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Logger interface for logging middleware.
|
|
134
|
+
*/
|
|
135
|
+
export interface LoggingLogger {
|
|
136
|
+
debug(message: string, data?: unknown): void;
|
|
137
|
+
info(message: string, data?: unknown): void;
|
|
138
|
+
warn(message: string, data?: unknown): void;
|
|
139
|
+
error(message: string, data?: unknown): void;
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Configuration for logging middleware.
|
|
143
|
+
*/
|
|
144
|
+
export interface LoggingMiddlewareConfig {
|
|
145
|
+
/**
|
|
146
|
+
* Minimum log level.
|
|
147
|
+
* @default 'info'
|
|
148
|
+
*/
|
|
149
|
+
level?: 'debug' | 'info' | 'warn' | 'error';
|
|
150
|
+
/**
|
|
151
|
+
* Whether to log request bodies.
|
|
152
|
+
* @default true
|
|
153
|
+
*/
|
|
154
|
+
logRequests?: boolean;
|
|
155
|
+
/**
|
|
156
|
+
* Whether to log response bodies.
|
|
157
|
+
* @default true
|
|
158
|
+
*/
|
|
159
|
+
logResponses?: boolean;
|
|
160
|
+
/**
|
|
161
|
+
* Whether to log errors.
|
|
162
|
+
* @default true
|
|
163
|
+
*/
|
|
164
|
+
logErrors?: boolean;
|
|
165
|
+
/**
|
|
166
|
+
* Whether to sanitize sensitive data (API keys, tokens).
|
|
167
|
+
* @default true
|
|
168
|
+
*/
|
|
169
|
+
sanitize?: boolean;
|
|
170
|
+
/**
|
|
171
|
+
* Custom logger implementation.
|
|
172
|
+
* @default console
|
|
173
|
+
*/
|
|
174
|
+
logger?: LoggingLogger;
|
|
175
|
+
/**
|
|
176
|
+
* Custom log prefix.
|
|
177
|
+
*/
|
|
178
|
+
prefix?: string;
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* Configuration for caching middleware.
|
|
182
|
+
*/
|
|
183
|
+
export interface CachingMiddlewareConfig {
|
|
184
|
+
/**
|
|
185
|
+
* Cache key generator.
|
|
186
|
+
* @default JSON.stringify(request)
|
|
187
|
+
*/
|
|
188
|
+
keyGenerator?: (request: IRChatRequest) => string;
|
|
189
|
+
/**
|
|
190
|
+
* Cache TTL in milliseconds.
|
|
191
|
+
* @default 3600000 (1 hour)
|
|
192
|
+
*/
|
|
193
|
+
ttl?: number;
|
|
194
|
+
/**
|
|
195
|
+
* Maximum cache size.
|
|
196
|
+
* @default 1000
|
|
197
|
+
*/
|
|
198
|
+
maxSize?: number;
|
|
199
|
+
/**
|
|
200
|
+
* Cache storage implementation.
|
|
201
|
+
*/
|
|
202
|
+
storage?: CacheStorage;
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* Cache storage interface.
|
|
206
|
+
*/
|
|
207
|
+
export interface CacheStorage {
|
|
208
|
+
get(key: string): Promise<IRChatResponse | undefined>;
|
|
209
|
+
set(key: string, value: IRChatResponse, ttl?: number): Promise<void>;
|
|
210
|
+
has(key: string): Promise<boolean>;
|
|
211
|
+
delete(key: string): Promise<boolean>;
|
|
212
|
+
clear(): Promise<void>;
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Configuration for telemetry middleware.
|
|
216
|
+
*/
|
|
217
|
+
export interface TelemetryMiddlewareConfig {
|
|
218
|
+
/**
|
|
219
|
+
* Telemetry sink for sending metrics.
|
|
220
|
+
*/
|
|
221
|
+
sink: TelemetrySink;
|
|
222
|
+
/**
|
|
223
|
+
* Whether to track request counts.
|
|
224
|
+
* @default true
|
|
225
|
+
*/
|
|
226
|
+
trackCounts?: boolean;
|
|
227
|
+
/**
|
|
228
|
+
* Whether to track latencies.
|
|
229
|
+
* @default true
|
|
230
|
+
*/
|
|
231
|
+
trackLatencies?: boolean;
|
|
232
|
+
/**
|
|
233
|
+
* Whether to track errors.
|
|
234
|
+
* @default true
|
|
235
|
+
*/
|
|
236
|
+
trackErrors?: boolean;
|
|
237
|
+
/**
|
|
238
|
+
* Whether to track token usage.
|
|
239
|
+
* @default true
|
|
240
|
+
*/
|
|
241
|
+
trackTokens?: boolean;
|
|
242
|
+
}
|
|
243
|
+
/**
|
|
244
|
+
* Telemetry sink interface.
|
|
245
|
+
*/
|
|
246
|
+
export interface TelemetrySink {
|
|
247
|
+
/**
|
|
248
|
+
* Record a metric.
|
|
249
|
+
*/
|
|
250
|
+
recordMetric(name: string, value: number, tags?: Record<string, string>): void;
|
|
251
|
+
/**
|
|
252
|
+
* Record an event.
|
|
253
|
+
*/
|
|
254
|
+
recordEvent(name: string, data?: Record<string, unknown>): void;
|
|
255
|
+
}
|
|
256
|
+
//# sourceMappingURL=middleware.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"middleware.d.ts","sourceRoot":"","sources":["../../src/middleware.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC1F,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAMpD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;;OAGG;IACH,OAAO,EAAE,aAAa,CAAC;IAEvB;;OAEG;IACH,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;IAE9B;;;OAGG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,cAAc,CAAC;IAElC;;OAEG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAE9B;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAExC;;OAEG;IACH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEzC;;OAEG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,0BAA2B,SAAQ,iBAAiB;IACnE,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC;IAE3B;;OAEG;IACH,KAAK,CAAC,EAAE,aAAa,CAAC;IAEtB;;OAEG;IACH,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IAEjC;;OAEG;IACH,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;CAClC;AAMD;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC,CAAC;AAE3D;;GAEG;AACH,MAAM,MAAM,uBAAuB,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,CAAC;AAElE;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,CACvB,OAAO,EAAE,iBAAiB,EAC1B,IAAI,EAAE,cAAc,KACjB,OAAO,CAAC,cAAc,CAAC,CAAC;AAE7B;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,CAChC,OAAO,EAAE,0BAA0B,EACnC,IAAI,EAAE,uBAAuB,KAC1B,OAAO,CAAC,YAAY,CAAC,CAAC;AAM3B;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B;;;OAGG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAE3B;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB;;OAEG;IACH,QAAQ,CAAC,iBAAiB,EAAE,OAAO,CAAC;IAEpC;;OAEG;IACH,QAAQ,CAAC,gBAAgB,EAAE,OAAO,CAAC;IAEnC;;OAEG;IACH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEzC;;OAEG;IACH,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;IAEhC;;OAEG;IACH,QAAQ,CAAC,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;CACpD;AAMD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IAC7C,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IAC5C,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IAC5C,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;CAC9C;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;IAE5C;;;OAGG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB;;;OAGG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IAEvB;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;OAGG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;;OAGG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC;;;OAGG;IACH,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,aAAa,KAAK,MAAM,CAAC;IAElD;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,OAAO,CAAC,EAAE,YAAY,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,CAAC;IACtD,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACrE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACnC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACtC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC;;OAEG;IACH,IAAI,EAAE,aAAa,CAAC;IAEpB;;;OAGG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;OAGG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB;;;OAGG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B;;OAEG;IACH,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;IAE/E;;OAEG;IACH,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CACjE"}
|
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Model Runner Backend Types
|
|
3
|
+
*
|
|
4
|
+
* Type definitions for backends that run models locally via subprocess/binary execution.
|
|
5
|
+
* Supports stdio and HTTP communication patterns.
|
|
6
|
+
*
|
|
7
|
+
* @module
|
|
8
|
+
*/
|
|
9
|
+
import type { BackendAdapterConfig } from './adapters.js';
|
|
10
|
+
/**
|
|
11
|
+
* Communication method with the model runner process.
|
|
12
|
+
*/
|
|
13
|
+
export type CommunicationType = 'stdio' | 'http';
|
|
14
|
+
/**
|
|
15
|
+
* stdio communication configuration.
|
|
16
|
+
*/
|
|
17
|
+
export interface StdioCommunicationConfig {
|
|
18
|
+
type: 'stdio';
|
|
19
|
+
/**
|
|
20
|
+
* Input format for requests.
|
|
21
|
+
* @default 'json-lines'
|
|
22
|
+
*/
|
|
23
|
+
inputFormat?: 'json-lines' | 'raw-text';
|
|
24
|
+
/**
|
|
25
|
+
* Output format for responses.
|
|
26
|
+
* @default 'json-lines'
|
|
27
|
+
*/
|
|
28
|
+
outputFormat?: 'json-lines' | 'raw-text';
|
|
29
|
+
/**
|
|
30
|
+
* Delimiter for line-based protocols.
|
|
31
|
+
* @default '\n'
|
|
32
|
+
*/
|
|
33
|
+
delimiter?: string;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* HTTP communication configuration.
|
|
37
|
+
*/
|
|
38
|
+
export interface HttpCommunicationConfig {
|
|
39
|
+
type: 'http';
|
|
40
|
+
/**
|
|
41
|
+
* Base URL template (can include {port} placeholder).
|
|
42
|
+
* @example 'http://localhost:{port}'
|
|
43
|
+
*/
|
|
44
|
+
baseURL: string;
|
|
45
|
+
/**
|
|
46
|
+
* Health check endpoint.
|
|
47
|
+
* @default '/health'
|
|
48
|
+
*/
|
|
49
|
+
healthEndpoint?: string;
|
|
50
|
+
/**
|
|
51
|
+
* Chat completions endpoint.
|
|
52
|
+
* @default '/v1/chat/completions'
|
|
53
|
+
*/
|
|
54
|
+
chatEndpoint?: string;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Communication configuration union type.
|
|
58
|
+
*/
|
|
59
|
+
export type CommunicationConfig = StdioCommunicationConfig | HttpCommunicationConfig;
|
|
60
|
+
/**
|
|
61
|
+
* Process execution configuration.
|
|
62
|
+
*/
|
|
63
|
+
export interface ProcessConfig {
|
|
64
|
+
/**
|
|
65
|
+
* Command to execute (binary path or command name).
|
|
66
|
+
* @example '/usr/local/bin/llama-cpp-server'
|
|
67
|
+
* @example 'ollama'
|
|
68
|
+
*/
|
|
69
|
+
command: string;
|
|
70
|
+
/**
|
|
71
|
+
* Command-line arguments.
|
|
72
|
+
* Can include placeholders: {modelPath}, {port}, {contextSize}, etc.
|
|
73
|
+
* @example ['--model', '{modelPath}', '--ctx-size', '{contextSize}']
|
|
74
|
+
*/
|
|
75
|
+
args?: string[];
|
|
76
|
+
/**
|
|
77
|
+
* Environment variables for the process.
|
|
78
|
+
* @example { CUDA_VISIBLE_DEVICES: '0' }
|
|
79
|
+
*/
|
|
80
|
+
env?: Record<string, string>;
|
|
81
|
+
/**
|
|
82
|
+
* Working directory for the process.
|
|
83
|
+
*/
|
|
84
|
+
cwd?: string;
|
|
85
|
+
/**
|
|
86
|
+
* Capture stdout from the process.
|
|
87
|
+
* @default true
|
|
88
|
+
*/
|
|
89
|
+
captureStdout?: boolean;
|
|
90
|
+
/**
|
|
91
|
+
* Capture stderr from the process.
|
|
92
|
+
* @default true
|
|
93
|
+
*/
|
|
94
|
+
captureStderr?: boolean;
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Model file reference.
|
|
98
|
+
*/
|
|
99
|
+
export interface ModelReference {
|
|
100
|
+
/**
|
|
101
|
+
* Path to the model file.
|
|
102
|
+
* @example './models/llama-3.1-8b.gguf'
|
|
103
|
+
*/
|
|
104
|
+
path: string;
|
|
105
|
+
/**
|
|
106
|
+
* Model format/type.
|
|
107
|
+
* @example 'gguf', 'safetensors', 'pytorch'
|
|
108
|
+
*/
|
|
109
|
+
format?: string;
|
|
110
|
+
/**
|
|
111
|
+
* Model name/identifier.
|
|
112
|
+
*/
|
|
113
|
+
name?: string;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Runtime configuration for model execution.
|
|
117
|
+
*/
|
|
118
|
+
export interface RuntimeConfig {
|
|
119
|
+
/**
|
|
120
|
+
* Context size (max tokens in context window).
|
|
121
|
+
* @default 2048
|
|
122
|
+
*/
|
|
123
|
+
contextSize?: number;
|
|
124
|
+
/**
|
|
125
|
+
* Number of GPU layers to offload.
|
|
126
|
+
* -1 = all layers, 0 = CPU only
|
|
127
|
+
* @default 0
|
|
128
|
+
*/
|
|
129
|
+
gpuLayers?: number;
|
|
130
|
+
/**
|
|
131
|
+
* Number of threads to use.
|
|
132
|
+
* @default (CPU cores)
|
|
133
|
+
*/
|
|
134
|
+
threads?: number;
|
|
135
|
+
/**
|
|
136
|
+
* Batch size for prompt processing.
|
|
137
|
+
* @default 512
|
|
138
|
+
*/
|
|
139
|
+
batchSize?: number;
|
|
140
|
+
/**
|
|
141
|
+
* Keep model loaded in memory.
|
|
142
|
+
* @default true
|
|
143
|
+
*/
|
|
144
|
+
keepAlive?: boolean;
|
|
145
|
+
/**
|
|
146
|
+
* Memory map the model file.
|
|
147
|
+
* @default true
|
|
148
|
+
*/
|
|
149
|
+
mmap?: boolean;
|
|
150
|
+
/**
|
|
151
|
+
* Lock model in memory (prevent swapping).
|
|
152
|
+
* @default false
|
|
153
|
+
*/
|
|
154
|
+
mlock?: boolean;
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Lifecycle management configuration.
|
|
158
|
+
*/
|
|
159
|
+
export interface LifecycleConfig {
|
|
160
|
+
/**
|
|
161
|
+
* Automatically start the process on construction.
|
|
162
|
+
* @default false
|
|
163
|
+
*/
|
|
164
|
+
autoStart?: boolean;
|
|
165
|
+
/**
|
|
166
|
+
* Automatically stop the process on dispose.
|
|
167
|
+
* @default true
|
|
168
|
+
*/
|
|
169
|
+
autoStop?: boolean;
|
|
170
|
+
/**
|
|
171
|
+
* Timeout for process startup (milliseconds).
|
|
172
|
+
* @default 60000 (60 seconds)
|
|
173
|
+
*/
|
|
174
|
+
startupTimeout?: number;
|
|
175
|
+
/**
|
|
176
|
+
* Timeout for graceful shutdown (milliseconds).
|
|
177
|
+
* @default 5000 (5 seconds)
|
|
178
|
+
*/
|
|
179
|
+
shutdownTimeout?: number;
|
|
180
|
+
/**
|
|
181
|
+
* Restart the process if it crashes.
|
|
182
|
+
* @default false
|
|
183
|
+
*/
|
|
184
|
+
autoRestart?: boolean;
|
|
185
|
+
/**
|
|
186
|
+
* Maximum restart attempts.
|
|
187
|
+
* @default 3
|
|
188
|
+
*/
|
|
189
|
+
maxRestarts?: number;
|
|
190
|
+
/**
|
|
191
|
+
* Health check interval (milliseconds).
|
|
192
|
+
* 0 = disabled
|
|
193
|
+
* @default 0
|
|
194
|
+
*/
|
|
195
|
+
healthCheckInterval?: number;
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* Configuration for model runner backends.
|
|
199
|
+
*
|
|
200
|
+
* Extends BackendAdapterConfig with model runner-specific options.
|
|
201
|
+
*/
|
|
202
|
+
export interface ModelRunnerBackendConfig extends BackendAdapterConfig {
|
|
203
|
+
/**
|
|
204
|
+
* Model configuration.
|
|
205
|
+
*/
|
|
206
|
+
model: string | ModelReference;
|
|
207
|
+
/**
|
|
208
|
+
* Process execution configuration.
|
|
209
|
+
*/
|
|
210
|
+
process: ProcessConfig;
|
|
211
|
+
/**
|
|
212
|
+
* Communication configuration.
|
|
213
|
+
*/
|
|
214
|
+
communication: CommunicationConfig;
|
|
215
|
+
/**
|
|
216
|
+
* Runtime configuration for model execution.
|
|
217
|
+
*/
|
|
218
|
+
runtime?: RuntimeConfig;
|
|
219
|
+
/**
|
|
220
|
+
* Lifecycle management configuration.
|
|
221
|
+
*/
|
|
222
|
+
lifecycle?: LifecycleConfig;
|
|
223
|
+
/**
|
|
224
|
+
* Port number for HTTP communication.
|
|
225
|
+
* If not specified, an available port will be auto-discovered.
|
|
226
|
+
*/
|
|
227
|
+
port?: number;
|
|
228
|
+
/**
|
|
229
|
+
* Prompt template name or custom template.
|
|
230
|
+
* @example 'llama2', 'chatml', 'alpaca'
|
|
231
|
+
*/
|
|
232
|
+
promptTemplate?: string | PromptTemplate;
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* Prompt template for formatting messages.
|
|
236
|
+
*/
|
|
237
|
+
export interface PromptTemplate {
|
|
238
|
+
/**
|
|
239
|
+
* Template name/identifier.
|
|
240
|
+
*/
|
|
241
|
+
name: string;
|
|
242
|
+
/**
|
|
243
|
+
* System message wrapper.
|
|
244
|
+
* @example '<<SYS>>\n{content}\n<</SYS>>'
|
|
245
|
+
*/
|
|
246
|
+
systemTemplate?: string;
|
|
247
|
+
/**
|
|
248
|
+
* User message wrapper.
|
|
249
|
+
* @example '[INST] {content} [/INST]'
|
|
250
|
+
*/
|
|
251
|
+
userTemplate?: string;
|
|
252
|
+
/**
|
|
253
|
+
* Assistant message wrapper.
|
|
254
|
+
* @example '{content}'
|
|
255
|
+
*/
|
|
256
|
+
assistantTemplate?: string;
|
|
257
|
+
/**
|
|
258
|
+
* Beginning of sequence token.
|
|
259
|
+
*/
|
|
260
|
+
bosToken?: string;
|
|
261
|
+
/**
|
|
262
|
+
* End of sequence token.
|
|
263
|
+
*/
|
|
264
|
+
eosToken?: string;
|
|
265
|
+
/**
|
|
266
|
+
* Add generation prompt at the end.
|
|
267
|
+
*/
|
|
268
|
+
addGenerationPrompt?: boolean;
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* Events emitted by model runner backends.
|
|
272
|
+
*/
|
|
273
|
+
export interface ModelRunnerEvents {
|
|
274
|
+
/**
|
|
275
|
+
* Process is starting.
|
|
276
|
+
*/
|
|
277
|
+
starting: void;
|
|
278
|
+
/**
|
|
279
|
+
* Process is ready to accept requests.
|
|
280
|
+
*/
|
|
281
|
+
ready: void;
|
|
282
|
+
/**
|
|
283
|
+
* Process has stopped.
|
|
284
|
+
*/
|
|
285
|
+
stopped: void;
|
|
286
|
+
/**
|
|
287
|
+
* Process encountered an error.
|
|
288
|
+
*/
|
|
289
|
+
error: Error;
|
|
290
|
+
/**
|
|
291
|
+
* Process is restarting.
|
|
292
|
+
*/
|
|
293
|
+
restarting: {
|
|
294
|
+
attempt: number;
|
|
295
|
+
maxAttempts: number;
|
|
296
|
+
};
|
|
297
|
+
/**
|
|
298
|
+
* stdout output from process.
|
|
299
|
+
*/
|
|
300
|
+
stdout: string;
|
|
301
|
+
/**
|
|
302
|
+
* stderr output from process.
|
|
303
|
+
*/
|
|
304
|
+
stderr: string;
|
|
305
|
+
}
|
|
306
|
+
/**
|
|
307
|
+
* Statistics about the running process.
|
|
308
|
+
*/
|
|
309
|
+
export interface ModelRunnerStats {
|
|
310
|
+
/**
|
|
311
|
+
* Is the process currently running?
|
|
312
|
+
*/
|
|
313
|
+
isRunning: boolean;
|
|
314
|
+
/**
|
|
315
|
+
* Process ID (if running).
|
|
316
|
+
*/
|
|
317
|
+
pid?: number;
|
|
318
|
+
/**
|
|
319
|
+
* Uptime in milliseconds.
|
|
320
|
+
*/
|
|
321
|
+
uptime?: number;
|
|
322
|
+
/**
|
|
323
|
+
* Number of requests processed.
|
|
324
|
+
*/
|
|
325
|
+
requestCount: number;
|
|
326
|
+
/**
|
|
327
|
+
* Number of restart attempts.
|
|
328
|
+
*/
|
|
329
|
+
restartCount: number;
|
|
330
|
+
/**
|
|
331
|
+
* Memory usage (if available).
|
|
332
|
+
*/
|
|
333
|
+
memory?: {
|
|
334
|
+
rss: number;
|
|
335
|
+
heapTotal: number;
|
|
336
|
+
heapUsed: number;
|
|
337
|
+
external: number;
|
|
338
|
+
};
|
|
339
|
+
/**
|
|
340
|
+
* CPU usage percentage (if available).
|
|
341
|
+
*/
|
|
342
|
+
cpu?: number;
|
|
343
|
+
}
|
|
344
|
+
//# sourceMappingURL=model-runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-runner.d.ts","sourceRoot":"","sources":["../../src/model-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAM1D;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG,OAAO,GAAG,MAAM,CAAC;AAEjD;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,IAAI,EAAE,OAAO,CAAC;IAEd;;;OAGG;IACH,WAAW,CAAC,EAAE,YAAY,GAAG,UAAU,CAAC;IAExC;;;OAGG;IACH,YAAY,CAAC,EAAE,YAAY,GAAG,UAAU,CAAC;IAEzC;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,MAAM,CAAC;IAEb;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,wBAAwB,GAAG,uBAAuB,CAAC;AAMrF;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAEhB;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE7B;;OAEG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb;;;OAGG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IAExB;;;OAGG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAMD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;OAGG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAMD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;OAGG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;OAGG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAMD;;;;GAIG;AACH,MAAM,WAAW,wBAAyB,SAAQ,oBAAoB;IACpE;;OAEG;IACH,KAAK,EAAE,MAAM,GAAG,cAAc,CAAC;IAE/B;;OAEG;IACH,OAAO,EAAE,aAAa,CAAC;IAEvB;;OAEG;IACH,aAAa,EAAE,mBAAmB,CAAC;IAEnC;;OAEG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;IAExB;;OAEG;IACH,SAAS,CAAC,EAAE,eAAe,CAAC;IAE5B;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,GAAG,cAAc,CAAC;CAC1C;AAMD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B;AAMD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,QAAQ,EAAE,IAAI,CAAC;IAEf;;OAEG;IACH,KAAK,EAAE,IAAI,CAAC;IAEZ;;OAEG;IACH,OAAO,EAAE,IAAI,CAAC;IAEd;;OAEG;IACH,KAAK,EAAE,KAAK,CAAC;IAEb;;OAEG;IACH,UAAU,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IAErD;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;CAChB;AAMD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;OAEG;IACH,SAAS,EAAE,OAAO,CAAC;IAEnB;;OAEG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,MAAM,CAAC,EAAE;QACP,GAAG,EAAE,MAAM,CAAC;QACZ,SAAS,EAAE,MAAM,CAAC;QAClB,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IAEF;;OAEG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;CACd"}
|