@huggingface/tasks 0.19.85 → 0.19.86

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/dist/commonjs/eval.d.ts +36 -0
  2. package/dist/commonjs/eval.d.ts.map +1 -0
  3. package/dist/commonjs/eval.js +38 -0
  4. package/dist/commonjs/index.d.ts +1 -0
  5. package/dist/commonjs/index.d.ts.map +1 -1
  6. package/dist/commonjs/index.js +3 -1
  7. package/dist/commonjs/inference-providers.d.ts +1 -1
  8. package/dist/commonjs/inference-providers.d.ts.map +1 -1
  9. package/dist/commonjs/inference-providers.js +1 -0
  10. package/dist/commonjs/local-apps.d.ts +7 -0
  11. package/dist/commonjs/local-apps.d.ts.map +1 -1
  12. package/dist/commonjs/local-apps.js +37 -0
  13. package/dist/commonjs/local-apps.spec.js +14 -0
  14. package/dist/commonjs/model-data.d.ts +1 -0
  15. package/dist/commonjs/model-data.d.ts.map +1 -1
  16. package/dist/commonjs/tasks/chat-completion/inference.d.ts +4 -0
  17. package/dist/commonjs/tasks/chat-completion/inference.d.ts.map +1 -1
  18. package/dist/esm/eval.d.ts +36 -0
  19. package/dist/esm/eval.d.ts.map +1 -0
  20. package/dist/esm/eval.js +35 -0
  21. package/dist/esm/index.d.ts +1 -0
  22. package/dist/esm/index.d.ts.map +1 -1
  23. package/dist/esm/index.js +1 -0
  24. package/dist/esm/inference-providers.d.ts +1 -1
  25. package/dist/esm/inference-providers.d.ts.map +1 -1
  26. package/dist/esm/inference-providers.js +1 -0
  27. package/dist/esm/local-apps.d.ts +7 -0
  28. package/dist/esm/local-apps.d.ts.map +1 -1
  29. package/dist/esm/local-apps.js +37 -0
  30. package/dist/esm/local-apps.spec.js +14 -0
  31. package/dist/esm/model-data.d.ts +1 -0
  32. package/dist/esm/model-data.d.ts.map +1 -1
  33. package/dist/esm/tasks/chat-completion/inference.d.ts +4 -0
  34. package/dist/esm/tasks/chat-completion/inference.d.ts.map +1 -1
  35. package/package.json +1 -1
  36. package/src/eval.ts +35 -0
  37. package/src/index.ts +2 -0
  38. package/src/inference-providers.ts +1 -0
  39. package/src/local-apps.spec.ts +16 -0
  40. package/src/local-apps.ts +44 -0
  41. package/src/model-data.ts +1 -0
  42. package/src/tasks/chat-completion/inference.ts +4 -0
  43. package/src/tasks/chat-completion/spec/input.json +5 -0
package/dist/commonjs/eval.d.ts ADDED
@@ -0,0 +1,36 @@
1
+ /**
2
+ * List of supported Evaluation Frameworks supported in the `eval.yaml` file in benchmarks datasets.
3
+ */
4
+ export declare const EVALUATION_FRAMEWORKS: {
5
+ readonly "inspect-ai": {
6
+ readonly name: "inspect-ai";
7
+ readonly description: "Inspect AI is an open-source framework for large language model evaluations.";
8
+ readonly url: "https://inspect.aisi.org.uk/";
9
+ };
10
+ readonly "math-arena": {
11
+ readonly name: "math-arena";
12
+ readonly description: "MathArena is a platform for evaluation of LLMs on latest math competitions and olympiads.";
13
+ readonly url: "https://github.com/eth-sri/matharena";
14
+ };
15
+ readonly mteb: {
16
+ readonly name: "mteb";
17
+ readonly description: "Multimodal toolbox for evaluating embeddings and retrieval systems.";
18
+ readonly url: "https://github.com/embeddings-benchmark/mteb";
19
+ };
20
+ readonly "olmo-bench": {
21
+ readonly name: "olmo-bench";
22
+ readonly description: "olmOCR-Bench is a framework for evaluating document-level OCR of various tools.";
23
+ readonly url: "https://github.com/allenai/olmocr/tree/main/olmocr/bench";
24
+ };
25
+ readonly harbor: {
26
+ readonly name: "harbor";
27
+ readonly description: "Harbor is a framework for evaluating and optimizing agents and language models.";
28
+ readonly url: "https://github.com/laude-institute/harbor";
29
+ };
30
+ readonly archipelago: {
31
+ readonly name: "archipelago";
32
+ readonly description: "Archipelago is a system for running and evaluating AI agents against MCP applications.";
33
+ readonly url: "https://github.com/Mercor-Intelligence/archipelago";
34
+ };
35
+ };
36
+ //# sourceMappingURL=eval.d.ts.map
package/dist/commonjs/eval.d.ts.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../src/eval.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+BxB,CAAC"}
package/dist/commonjs/eval.js ADDED
@@ -0,0 +1,38 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.EVALUATION_FRAMEWORKS = void 0;
4
+ /**
5
+ * List of supported Evaluation Frameworks supported in the `eval.yaml` file in benchmarks datasets.
6
+ */
7
+ exports.EVALUATION_FRAMEWORKS = {
8
+ "inspect-ai": {
9
+ name: "inspect-ai",
10
+ description: "Inspect AI is an open-source framework for large language model evaluations.",
11
+ url: "https://inspect.aisi.org.uk/",
12
+ },
13
+ "math-arena": {
14
+ name: "math-arena",
15
+ description: "MathArena is a platform for evaluation of LLMs on latest math competitions and olympiads.",
16
+ url: "https://github.com/eth-sri/matharena",
17
+ },
18
+ mteb: {
19
+ name: "mteb",
20
+ description: "Multimodal toolbox for evaluating embeddings and retrieval systems.",
21
+ url: "https://github.com/embeddings-benchmark/mteb",
22
+ },
23
+ "olmo-bench": {
24
+ name: "olmo-bench",
25
+ description: "olmOCR-Bench is a framework for evaluating document-level OCR of various tools.",
26
+ url: "https://github.com/allenai/olmocr/tree/main/olmocr/bench",
27
+ },
28
+ harbor: {
29
+ name: "harbor",
30
+ description: "Harbor is a framework for evaluating and optimizing agents and language models.",
31
+ url: "https://github.com/laude-institute/harbor",
32
+ },
33
+ archipelago: {
34
+ name: "archipelago",
35
+ description: "Archipelago is a system for running and evaluating AI agents against MCP applications.",
36
+ url: "https://github.com/Mercor-Intelligence/archipelago",
37
+ },
38
+ };
package/dist/commonjs/index.d.ts CHANGED
@@ -18,4 +18,5 @@ export type { LocalApp, LocalAppKey, LocalAppSnippet } from "./local-apps.js";
18
18
  export { DATASET_LIBRARIES_UI_ELEMENTS } from "./dataset-libraries.js";
19
19
  export type { DatasetLibraryUiElement, DatasetLibraryKey } from "./dataset-libraries.js";
20
20
  export * from "./inference-providers.js";
21
+ export { EVALUATION_FRAMEWORKS } from "./eval.js";
21
22
  //# sourceMappingURL=index.d.ts.map
package/dist/commonjs/index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AACpE,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACvF,cAAc,kBAAkB,CAAC;AACjC,OAAO,EACN,aAAa,EACb,cAAc,EACd,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,QAAQ,EACb,UAAU,EACV,eAAe,EACf,aAAa,EACb,kBAAkB,GAClB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACN,8BAA8B,EAC9B,sBAAsB,EACtB,2BAA2B,GAC3B,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC9E,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnE,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACzF,YAAY,EACX,aAAa,EACb,sBAAsB,EACtB,gCAAgC,EAChC,8BAA8B,EAC9B,kCAAkC,EAClC,uBAAuB,EACvB,sBAAsB,EACtB,oCAAoC,EACpC,gCAAgC,EAChC,2BAA2B,EAC3B,gCAAgC,EAChC,8BAA8B,EAC9B,sBAAsB,EACtB,8BAA8B,EAC9B,mBAAmB,EACnB,sBAAsB,EACtB,yBAAyB,EACzB,8BAA8B,EAC9B,uBAAuB,GACvB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAC;AAEhE,cAAc,WAAW,CAAC;AAE1B,OAAO,EACN,KAAK,gBAAgB,EACrB,KAAK,wBAAwB,EAC7B,KAAK,gBAAgB,EACrB,yBAAyB,EACzB,yBAAyB,EACzB,iBAAiB,EACjB,oBAAoB,GACpB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,IAAI,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAC7D,YAAY,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAE9E,OAAO,EAAE,6BAA6B,EAAE,MAAM,wBAAwB,CAAC;AACvE,YAAY,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAEzF,cAAc,0BAA0B,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AACpE,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACvF,cAAc,kBAAkB,CAAC;AACjC,OAAO,EACN,aAAa,EACb,cAAc,EACd,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,QAAQ,EACb,UAAU,EACV,eAAe,EACf,aAAa,EACb,kBAAkB,GAClB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACN,8BAA8B,EAC9B,sBAAsB,EACtB,2BAA2B,GAC3B,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC9E,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnE,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACzF,YAAY,EACX,aAAa,EACb,sBAAsB,EACtB,gCAAgC,EAChC,8BAA8B,EAC9B,kCAAkC,EAClC,uBAAuB,EACvB,sBAAsB,EACtB,oCAAoC,EACpC,gCAAgC,EAChC,2BAA2B,EAC3B,gCAAgC,EAChC,8BAA8B,EAC9B,sBAAsB,EACtB,8BAA8B,EAC9B,mBAAmB,EACnB,sBAAsB,EACtB,yBAAyB,EACzB,8BAA8B,EAC9B,uBAAuB,GACvB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAC;AAEhE,cAAc,WAAW,CAAC;AAE1B,OAAO,EACN,KAAK,gBAAgB,EACrB,KAAK,wBAAwB,EAC7B,KAAK,gBAAgB,EACrB,yBAAyB,EACzB,yBAAyB,EACzB,iBAAiB,EACjB,oBAAoB,GACpB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,IAAI,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAC7D,YAAY,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAE9E,OAAO,EAAE,6BAA6B,EAAE,MAAM,wBAAwB,CAAC;AACvE,YAAY,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAEzF,cAAc,0BAA0B,CAAC;AAEzC,OAAO,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC"}
package/dist/commonjs/index.js CHANGED
@@ -14,7 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
14
  for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
15
  };
16
16
  Object.defineProperty(exports, "__esModule", { value: true });
17
- exports.DATASET_LIBRARIES_UI_ELEMENTS = exports.LOCAL_APPS = exports.DEFAULT_MEMORY_OPTIONS = exports.SKUS = exports.getModelInputSnippet = exports.stringifyMessages = exports.stringifyGenerationConfig = exports.inferenceSnippetLanguages = exports.SPECIAL_TOKENS_ATTRIBUTES = exports.MODEL_LIBRARIES_UI_ELEMENTS = exports.ALL_MODEL_LIBRARY_KEYS = exports.ALL_DISPLAY_MODEL_LIBRARY_KEYS = exports.PIPELINE_TYPES_SET = exports.SUBTASK_TYPES = exports.MODALITY_LABELS = exports.MODALITIES = exports.PIPELINE_TYPES = exports.PIPELINE_DATA = exports.MAPPING_DEFAULT_WIDGET = exports.LIBRARY_TASK_MAPPING = void 0;
17
+ exports.EVALUATION_FRAMEWORKS = exports.DATASET_LIBRARIES_UI_ELEMENTS = exports.LOCAL_APPS = exports.DEFAULT_MEMORY_OPTIONS = exports.SKUS = exports.getModelInputSnippet = exports.stringifyMessages = exports.stringifyGenerationConfig = exports.inferenceSnippetLanguages = exports.SPECIAL_TOKENS_ATTRIBUTES = exports.MODEL_LIBRARIES_UI_ELEMENTS = exports.ALL_MODEL_LIBRARY_KEYS = exports.ALL_DISPLAY_MODEL_LIBRARY_KEYS = exports.PIPELINE_TYPES_SET = exports.SUBTASK_TYPES = exports.MODALITY_LABELS = exports.MODALITIES = exports.PIPELINE_TYPES = exports.PIPELINE_DATA = exports.MAPPING_DEFAULT_WIDGET = exports.LIBRARY_TASK_MAPPING = void 0;
18
18
  var library_to_tasks_js_1 = require("./library-to-tasks.js");
19
19
  Object.defineProperty(exports, "LIBRARY_TASK_MAPPING", { enumerable: true, get: function () { return library_to_tasks_js_1.LIBRARY_TASK_MAPPING; } });
20
20
  var default_widget_inputs_js_1 = require("./default-widget-inputs.js");
@@ -47,3 +47,5 @@ Object.defineProperty(exports, "LOCAL_APPS", { enumerable: true, get: function (
47
47
  var dataset_libraries_js_1 = require("./dataset-libraries.js");
48
48
  Object.defineProperty(exports, "DATASET_LIBRARIES_UI_ELEMENTS", { enumerable: true, get: function () { return dataset_libraries_js_1.DATASET_LIBRARIES_UI_ELEMENTS; } });
49
49
  __exportStar(require("./inference-providers.js"), exports);
50
+ var eval_js_1 = require("./eval.js");
51
+ Object.defineProperty(exports, "EVALUATION_FRAMEWORKS", { enumerable: true, get: function () { return eval_js_1.EVALUATION_FRAMEWORKS; } });
package/dist/commonjs/inference-providers.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- declare const INFERENCE_PROVIDERS: readonly ["cerebras", "cohere", "fal-ai", "fireworks-ai", "hf-inference", "hyperbolic", "ovhcloud", "replicate", "sambanova", "together"];
1
+ declare const INFERENCE_PROVIDERS: readonly ["cerebras", "cohere", "deepinfra", "fal-ai", "fireworks-ai", "hf-inference", "hyperbolic", "ovhcloud", "replicate", "sambanova", "together"];
2
2
  export type SnippetInferenceProvider = (typeof INFERENCE_PROVIDERS)[number] | string;
3
3
  export declare const HF_HUB_INFERENCE_PROXY_TEMPLATE = "https://router.huggingface.co/{{PROVIDER}}";
4
4
  /**
package/dist/commonjs/inference-providers.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"inference-providers.d.ts","sourceRoot":"","sources":["../../src/inference-providers.ts"],"names":[],"mappings":"AAEA,QAAA,MAAM,mBAAmB,2IAWf,CAAC;AAEX,MAAM,MAAM,wBAAwB,GAAG,CAAC,OAAO,mBAAmB,CAAC,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;AAErF,eAAO,MAAM,+BAA+B,+CAA+C,CAAC;AAE5F;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,wBAAwB,GAAG,MAAM,CAGxE"}
1
+ {"version":3,"file":"inference-providers.d.ts","sourceRoot":"","sources":["../../src/inference-providers.ts"],"names":[],"mappings":"AAEA,QAAA,MAAM,mBAAmB,wJAYf,CAAC;AAEX,MAAM,MAAM,wBAAwB,GAAG,CAAC,OAAO,mBAAmB,CAAC,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;AAErF,eAAO,MAAM,+BAA+B,+CAA+C,CAAC;AAE5F;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,wBAAwB,GAAG,MAAM,CAGxE"}
package/dist/commonjs/inference-providers.js CHANGED
@@ -7,6 +7,7 @@ exports.openAIbaseUrl = openAIbaseUrl;
7
7
  const INFERENCE_PROVIDERS = [
8
8
  "cerebras",
9
9
  "cohere",
10
+ "deepinfra",
10
11
  "fal-ai",
11
12
  "fireworks-ai",
12
13
  "hf-inference",
package/dist/commonjs/local-apps.d.ts CHANGED
@@ -212,6 +212,13 @@ export declare const LOCAL_APPS: {
212
212
  displayOnModelPage: (model: ModelData) => boolean;
213
213
  snippet: (model: ModelData, filepath?: string) => LocalAppSnippet[];
214
214
  };
215
+ pi: {
216
+ prettyLabel: string;
217
+ docsUrl: string;
218
+ mainTask: "text-generation";
219
+ displayOnModelPage: (model: ModelData) => boolean;
220
+ snippet: (model: ModelData, filepath?: string) => LocalAppSnippet[];
221
+ };
215
222
  };
216
223
  export type LocalAppKey = keyof typeof LOCAL_APPS;
217
224
  export {};
package/dist/commonjs/local-apps.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"local-apps.d.ts","sourceRoot":"","sources":["../../src/local-apps.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAKnD,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG;IACtB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,QAAQ,EAAE,YAAY,CAAC;IACvB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;OAEG;IACH,kBAAkB,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,CAAC;CAClD,GAAG,CACD;IACA;;OAEG;IACH,QAAQ,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,GAAG,CAAC;CACtD,GACD;IACA;;;;OAIG;IACH,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM,EAAE,GAAG,eAAe,GAAG,eAAe,EAAE,CAAC;CACzG,CACH,CAAC;AAsBF,iBAAS,UAAU,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAE7C;AAED,iBAAS,mBAAmB,CAAC,KAAK,EAAE,SAAS,WAE5C;AAED,iBAAS,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAU9C;AAED,iBAAS,wBAAwB,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAE3D;AA4UD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,UAAU;;;;;;yBAlUS,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBAiDzC,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBAyC3D,SAAS,KAAG,eAAe,EAAE;;;;;;oCAkQ3B,SAAS;yBA5MT,SAAS,KAAG,eAAe,EAAE;;;;;;;yBAoF9B,SAAS,KAAG,eAAe,EAAE;;;;;;;yBA7B/B,SAAS,KAAG,eAAe,EAAE;;;;;;;;;;;;;;yBApIzB,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yBAJjD,SAAS,aAAa,MAAM,KAAG,MAAM;;;;;;;yBAwM1B,SAAS,aAAa,MAAM,KAAG,MAAM;;;;;;;yBAM9C,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;CAiO5C,CAAC;AAErC,MAAM,MAAM,WAAW,GAAG,MAAM,OAAO,UAAU,CAAC"}
1
+ {"version":3,"file":"local-apps.d.ts","sourceRoot":"","sources":["../../src/local-apps.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAKnD,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG;IACtB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,QAAQ,EAAE,YAAY,CAAC;IACvB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;OAEG;IACH,kBAAkB,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,CAAC;CAClD,GAAG,CACD;IACA;;OAEG;IACH,QAAQ,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,GAAG,CAAC;CACtD,GACD;IACA;;;;OAIG;IACH,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM,EAAE,GAAG,eAAe,GAAG,eAAe,EAAE,CAAC;CACzG,CACH,CAAC;AAsBF,iBAAS,UAAU,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAE7C;AAED,iBAAS,mBAAmB,CAAC,KAAK,EAAE,SAAS,WAE5C;AAED,iBAAS,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAU9C;AAED,iBAAS,wBAAwB,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAE3D;AAiXD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,UAAU;;;;;;yBAvWS,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBAiDzC,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBAyC3D,SAAS,KAAG,eAAe,EAAE;;;;;;oCAuS3B,SAAS;yBAjPT,SAAS,KAAG,eAAe,EAAE;;;;;;;yBAoF9B,SAAS,KAAG,eAAe,EAAE;;;;;;;yBA7B/B,SAAS,KAAG,eAAe,EAAE;;;;;;;;;;;;;;yBApIzB,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yBAJjD,SAAS,aAAa,MAAM,KAAG,MAAM;;;;;;;yBA6O1B,SAAS,aAAa,MAAM,KAAG,MAAM;;;;;;;yBAM9C,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBA3CtD,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;CAmRtC,CAAC;AAErC,MAAM,MAAM,WAAW,GAAG,MAAM,OAAO,UAAU,CAAC"}
package/dist/commonjs/local-apps.js CHANGED
@@ -297,6 +297,36 @@ const snippetMlxLm = (model) => {
297
297
  : []),
298
298
  ];
299
299
  };
300
+ const snippetPi = (model, filepath) => {
301
+ const quantTag = getQuantTag(filepath);
302
+ const modelName = model.id.split("/").pop() ?? model.id;
303
+ const modelsJson = JSON.stringify({
304
+ providers: {
305
+ "llama-cpp": {
306
+ baseUrl: "http://localhost:8080/v1",
307
+ api: "openai-completions",
308
+ apiKey: "none",
309
+ models: [{ id: modelName }],
310
+ },
311
+ },
312
+ }, null, 2);
313
+ return [
314
+ {
315
+ title: "Start the llama.cpp server",
316
+ setup: "# Install llama.cpp:\nbrew install llama.cpp",
317
+ content: `# Start a local OpenAI-compatible server:\nllama-server -hf ${model.id}${quantTag} --jinja`,
318
+ },
319
+ {
320
+ title: "Configure the model in Pi",
321
+ setup: "# Install Pi:\nnpm install -g @mariozechner/pi-coding-agent",
322
+ content: `# Add to ~/.pi/agent/models.json:\n${modelsJson}`,
323
+ },
324
+ {
325
+ title: "Run Pi",
326
+ content: `# Start Pi in your project directory:\npi`,
327
+ },
328
+ ];
329
+ };
300
330
  const snippetDockerModelRunner = (model, filepath) => {
301
331
  // Only add quant tag for GGUF models, not safetensors
302
332
  const quantTag = isLlamaCppGgufModel(model) ? getQuantTag(filepath) : "";
@@ -523,4 +553,11 @@ exports.LOCAL_APPS = {
523
553
  displayOnModelPage: (model) => isLlamaCppGgufModel(model) || isAmdRyzenModel(model),
524
554
  snippet: snippetLemonade,
525
555
  },
556
+ pi: {
557
+ prettyLabel: "Pi",
558
+ docsUrl: "https://github.com/badlogic/pi-mono",
559
+ mainTask: "text-generation",
560
+ displayOnModelPage: (model) => isLlamaCppGgufModel(model) && !!model.gguf?.chat_template?.includes("tools"),
561
+ snippet: snippetPi,
562
+ },
526
563
  };
package/dist/commonjs/local-apps.spec.js CHANGED
@@ -112,6 +112,20 @@ curl -X POST "http://localhost:8000/v1/chat/completions" \\
112
112
  ]
113
113
  }'`);
114
114
  });
115
+ (0, vitest_1.it)("pi", async () => {
116
+ const { snippet: snippetFunc } = local_apps_js_1.LOCAL_APPS["pi"];
117
+ const model = {
118
+ id: "bartowski/Llama-3.2-3B-Instruct-GGUF",
119
+ tags: ["conversational"],
120
+ gguf: { total: 1, context_length: 4096, chat_template: "{% if tools %}" },
121
+ inference: "",
122
+ };
123
+ const snippet = snippetFunc(model);
124
+ (0, vitest_1.expect)(snippet[0].content).toContain(`llama-server -hf bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}} --jinja`);
125
+ (0, vitest_1.expect)(snippet[1].setup).toContain("npm install -g @mariozechner/pi-coding-agent");
126
+ (0, vitest_1.expect)(snippet[1].content).toContain(`"id": "Llama-3.2-3B-Instruct-GGUF"`);
127
+ (0, vitest_1.expect)(snippet[2].content).toContain("pi");
128
+ });
115
129
  (0, vitest_1.it)("docker model runner", async () => {
116
130
  const { snippet: snippetFunc } = local_apps_js_1.LOCAL_APPS["docker-model-runner"];
117
131
  const model = {
package/dist/commonjs/model-data.d.ts CHANGED
@@ -127,6 +127,7 @@ export interface ModelData {
127
127
  total: number;
128
128
  architecture?: string;
129
129
  context_length?: number;
130
+ chat_template?: string;
130
131
  };
131
132
  }
132
133
  /**
package/dist/commonjs/model-data.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"model-data.d.ts","sourceRoot":"","sources":["../../src/model-data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAE3D;;GAEG;AACH,MAAM,WAAW,SAAS;IACzB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IACX;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;OAEG;IACH,MAAM,CAAC,EAAE;QACR,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;QACzB;;WAEG;QACH,QAAQ,CAAC,EAAE;YACV;;eAEG;YACH,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;SACpB,CAAC;QACF,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,mBAAmB,CAAC,EAAE;YACrB,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,YAAY,CAAC,EAAE,OAAO,CAAC;YACvB,YAAY,CAAC,EAAE,OAAO,CAAC;YACvB;;eAEG;YACH,YAAY,CAAC,EAAE,MAAM,CAAC;SACtB,CAAC;QACF,gBAAgB,CAAC,EAAE,eAAe,CAAC;QACnC,gBAAgB,CAAC,EAAE;YAClB,aAAa,CAAC,EAAE,MAAM,CAAC;SACvB,CAAC;QACF,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,oBAAoB,CAAC,EAAE;YACtB,UAAU,CAAC,EAAE,MAAM,CAAC;YACpB,WAAW,CAAC,EAAE,MAAM,CAAC;SACrB,CAAC;QACF,SAAS,CAAC,EAAE;YACX,WAAW,CAAC,EAAE,MAAM,CAAC;SACrB,CAAC;QACF,OAAO,CAAC,EAAE;YACT,KAAK,CAAC,EAAE;gBACP,IAAI,CAAC,EAAE,MAAM,CAAC;aACd,CAAC;YACF,YAAY,CAAC,EAAE,MAAM,CAAC;SACtB,CAAC;QACF,WAAW,CAAC,EAAE;YACb,qBAAqB,CAAC,EAAE,MAAM,CAAC;YAC/B,iBAAiB,CAAC,EAAE,MAAM,CAAC;YAC3B,gBAAgB,CAAC,EAAE,MAAM,CAAC;SAC1B,CAAC;QACF,IAAI,CAAC,EAAE;YACN,uBAAuB,CAAC,EAAE,MAAM,CAAC;YACjC,SAAS,CAAC,EAAE,MAAM,CAAC;SACnB,CAAC;QACF,SAAS,CAAC,EAAE;YACX,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;SACjB,CAAC;KACF,CAAC;IACF;;OAEG;IACH,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;;OAEG;IACH,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC;;OAEG;IACH,YAAY,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;IACxC;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC;;;;;OAKG;IACH,UAAU,CAAC,EAAE,aAAa,EAAE,GAAG,SAAS,CAAC;IACzC;;;;;;;;;OASG;IACH,QAAQ,CAAC,EAAE;QACV,SAAS,CAAC,EACP,OAAO,GACP;YACA,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;SACpC,CAAC;QACL,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;QAC/B,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;KAChC,CAAC;IACF;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE;QACb,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACnC,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,OAAO,CAAC;KACjB,CAAC;IACF,IAAI,CAAC,EAAE;QACN,KAAK,EAAE,MAAM,CAAC;QACd,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,cAAc,CAAC,EAAE,MAAM,CAAC;KACxB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB"}
1
+ {"version":3,"file":"model-data.d.ts","sourceRoot":"","sources":["../../src/model-data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAE3D;;GAEG;AACH,MAAM,WAAW,SAAS;IACzB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IACX;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;OAEG;IACH,MAAM,CAAC,EAAE;QACR,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;QACzB;;WAEG;QACH,QAAQ,CAAC,EAAE;YACV;;eAEG;YACH,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;SACpB,CAAC;QACF,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,mBAAmB,CAAC,EAAE;YACrB,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,YAAY,CAAC,EAAE,OAAO,CAAC;YACvB,YAAY,CAAC,EAAE,OAAO,CAAC;YACvB;;eAEG;YACH,YAAY,CAAC,EAAE,MAAM,CAAC;SACtB,CAAC;QACF,gBAAgB,CAAC,EAAE,eAAe,CAAC;QACnC,gBAAgB,CAAC,EAAE;YAClB,aAAa,CAAC,EAAE,MAAM,CAAC;SACvB,CAAC;QACF,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,oBAAoB,CAAC,EAAE;YACtB,UAAU,CAAC,EAAE,MAAM,CAAC;YACpB,WAAW,CAAC,EAAE,MAAM,CAAC;SACrB,CAAC;QACF,SAAS,CAAC,EAAE;YACX,WAAW,CAAC,EAAE,MAAM,CAAC;SACrB,CAAC;QACF,OAAO,CAAC,EAAE;YACT,KAAK,CAAC,EAAE;gBACP,IAAI,CAAC,EAAE,MAAM,CAAC;aACd,CAAC;YACF,YAAY,CAAC,EAAE,MAAM,CAAC;SACtB,CAAC;QACF,WAAW,CAAC,EAAE;YACb,qBAAqB,CAAC,EAAE,MAAM,CAAC;YAC/B,iBAAiB,CAAC,EAAE,MAAM,CAAC;YAC3B,gBAAgB,CAAC,EAAE,MAAM,CAAC;SAC1B,CAAC;QACF,IAAI,CAAC,EAAE;YACN,uBAAuB,CAAC,EAAE,MAAM,CAAC;YACjC,SAAS,CAAC,EAAE,MAAM,CAAC;SACnB,CAAC;QACF,SAAS,CAAC,EAAE;YACX,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;SACjB,CAAC;KACF,CAAC;IACF;;OAEG;IACH,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;;OAEG;IACH,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC;;OAEG;IACH,YAAY,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;IACxC;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC;;;;;OAKG;IACH,UAAU,CAAC,EAAE,aAAa,EAAE,GAAG,SAAS,CAAC;IACzC;;;;;;;;;OASG;IACH,QAAQ,CAAC,EAAE;QACV,SAAS,CAAC,EACP,OAAO,GACP;YACA,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;SACpC,CAAC;QACL,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;QAC/B,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;KAChC,CAAC;IACF;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE;QACb,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACnC,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,OAAO,CAAC;KACjB,CAAC;IACF,IAAI,CAAC,EAAE;QACN,KAAK,EAAE,MAAM,CAAC;QACd,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,aAAa,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB"}
package/dist/commonjs/tasks/chat-completion/inference.d.ts CHANGED
@@ -62,6 +62,10 @@ export interface ChatCompletionInput {
62
62
  * increasing the model's likelihood to talk about new topics
63
63
  */
64
64
  presence_penalty?: number;
65
+ /**
66
+ * Optional. Constrains effort on reasoning for reasoning models. Reducing reasoning effort can result in faster responses and fewer tokens used on reasoning. Common values: none, minimal, low, medium, high, xhigh. Support and defaults are provider and model-dependent.
67
+ */
68
+ reasoning_effort?: string;
65
69
  response_format?: ChatCompletionInputGrammarType;
66
70
  seed?: number;
67
71
  /**
package/dist/commonjs/tasks/chat-completion/inference.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/chat-completion/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;;;;;GAMG;AACH,MAAM,WAAW,mBAAmB;IACnC;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;;;;;;;;;OAWG;IACH,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,QAAQ,EAAE,0BAA0B,EAAE,CAAC;IACvC;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;OAKG;IACH,CAAC,CAAC,EAAE,MAAM,CAAC;IACX;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,8BAA8B,CAAC;IACjD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,cAAc,CAAC,EAAE,gCAAgC,CAAC;IAClD;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,6BAA6B,CAAC;IAC5C;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,KAAK,CAAC,EAAE,uBAAuB,EAAE,CAAC;IAClC;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;OAKG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,0BAA0B;IAC1C,OAAO,CAAC,EAAE,iCAAiC,CAAC;IAC5C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,2BAA2B,EAAE,CAAC;IAC3C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,iCAAiC,GAAG,+BAA+B,EAAE,GAAG,MAAM,CAAC;AAC3F,MAAM,WAAW,+BAA+B;IAC/C,SAAS,CAAC,EAAE,sBAAsB,CAAC;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,mCAAmC,CAAC;IAC1C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,sBAAsB;IACtC,GAAG,EAAE,MAAM,CAAC;IACZ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,mCAAmC,GAAG,MAAM,GAAG,WAAW,CAAC;AACvE,MAAM,WAAW,2BAA2B;IAC3C,QAAQ,EAAE,qCAAqC,CAAC;IAChD,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,qCAAqC;IACrD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,8BAA8B;IAC9C,WAAW,CAAC,EAAE,mCAAmC,CAAC;IAClD,IAAI,EAAE,kCAAkC,CAAC;IACzC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,mCAAmC;IACnD;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,MAAM,CAAC,EAAE;QACR,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACvB,CAAC;IACF;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,kCAAkC,GAAG,MAAM,GAAG,aAAa,GAAG,aAAa,CAAC;AACxF,MAAM,WAAW,gCAAgC;IAChD;;;;;OAKG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;GAGG;AACH,MAAM,MAAM,6BAA6B,GAAG,iCAAiC,GAAG,mCAAmC,CAAC;AACpH;;;;;;GAMG;AACH,MAAM,MAAM,iCAAiC,GAAG,MAAM,GAAG,MAAM,GAAG,UAAU,CAAC;AAC7E,MAAM,WAAW,mCAAmC;IACnD,QAAQ,EAAE,+BAA+B,CAAC;IAC1C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,+BAA+B;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,uBAAuB;IACvC,QAAQ,EAAE,qCAAqC,CAAC;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB;IACpC,OAAO,EAAE,4BAA4B,EAAE,CAAC;IACxC,OAAO,EAAE,MAAM,CAAC;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,EAAE,yBAAyB,CAAC;IACjC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,4BAA4B;IAC5C,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,4BAA4B,CAAC;IACxC,OAAO,EAAE,2BAA2B,CAAC;IACrC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,4BAA4B;IAC5C,OAAO,EAAE,2BAA2B,EAAE,CAAC;IACvC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,2BAA2B;IAC3C,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,8BAA8B,EAAE,CAAC;IAC/C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,8BAA8B;IAC9C,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,2BAA2B;IAC3C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,4BAA4B,EAAE,CAAC;IAC5C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,4BAA4B;IAC5C,QAAQ,EAAE,sCAAsC,CAAC;IACjD,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,sCAAsC;IACtD,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,yBAAyB;IACzC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;;;;GAMG;AACH,MAAM,WAAW,0BAA0B;IAC1C,OAAO,EAAE,gCAAgC,EAAE,CAAC;IAC5C,OAAO,EAAE,MAAM,CAAC;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,CAAC,EAAE,+BAA+B,CAAC;IACxC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,gCAAgC;IAChD,KAAK,EAAE,+BAA+B,CAAC;IACvC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,kCAAkC,CAAC;IAC9C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,+BAA+B;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,uCAAuC,EAAE,CAAC;IACvD,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,uCAAuC;IACvD,QAAQ,EAAE,kCAAkC,CAAC;IAC7C,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,kCAAkC;IAClD,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,kCAAkC;IAClD,OAAO,EAAE,iCAAiC,EAAE,CAAC;IAC7C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,iCAAiC;IACjD,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,oCAAoC,EAAE,CAAC;IACrD,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,oCAAoC;IACpD,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,+BAA+B;IAC/C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"}
1
+ {"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/chat-completion/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;;;;;GAMG;AACH,MAAM,WAAW,mBAAmB;IACnC;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;;;;;;;;;OAWG;IACH,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,QAAQ,EAAE,0BAA0B,EAAE,CAAC;IACvC;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;OAKG;IACH,CAAC,CAAC,EAAE,MAAM,CAAC;IACX;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,8BAA8B,CAAC;IACjD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,cAAc,CAAC,EAAE,gCAAgC,CAAC;IAClD;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,6BAA6B,CAAC;IAC5C;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,KAAK,CAAC,EAAE,uBAAuB,EAAE,CAAC;IAClC;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;OAKG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,0BAA0B;IAC1C,OAAO,CAAC,EAAE,iCAAiC,CAAC;IAC5C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,2BAA2B,EAAE,CAAC;IAC3C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,iCAAiC,GAAG,+BAA+B,EAAE,GAAG,MAAM,CAAC;AAC3F,MAAM,WAAW,+BAA+B;IAC/C,SAAS,CAAC,EAAE,sBAAsB,CAAC;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,mCAAmC,CAAC;IAC1C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,sBAAsB;IACtC,GAAG,EAAE,MAAM,CAAC;IACZ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,mCAAmC,GAAG,MAAM,GAAG,WAAW,CAAC;AACvE,MAAM,WAAW,2BAA2B;IAC3C,QAAQ,EAAE,qCAAqC,CAAC;IAChD,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,qCAAqC;IACrD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,8BAA8B;IAC9C,WAAW,CAAC,EAAE,mCAAmC,CAAC;IAClD,IAAI,EAAE,kCAAkC,CAAC;IACzC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,mCAAmC;IACnD;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,MAAM,CAAC,EAAE;QACR,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACvB,CAAC;IACF;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,kCAAkC,GAAG,MAAM,GAAG,aAAa,GAAG,aAAa,CAAC;AACxF,MAAM,WAAW,gCAAgC;IAChD;;;;;OAKG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;GAGG;AACH,MAAM,MAAM,6BAA6B,GAAG,iCAAiC,GAAG,mCAAmC,CAAC;AACpH;;;;;;GAMG;AACH,MAAM,MAAM,iCAAiC,GAAG,MAAM,GAAG,MAAM,GAAG,UAAU,CAAC;AAC7E,MAAM,WAAW,mCAAmC;IACnD,QAAQ,EAAE,+BAA+B,CAAC;IAC1C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,+BAA+B;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,uBAAuB;IACvC,QAAQ,EAAE,qCAAqC,CAAC;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB;IACpC,OAAO,EAAE,4BAA4B,EAAE,CAAC;IACxC,OAAO,EAAE,MAAM,CAAC;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,EAAE,yBAAyB,CAAC;IACjC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,4BAA4B;IAC5C,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,4BAA4B,CAAC;IACxC,OAAO,EAAE,2BAA2B,CAAC;IACrC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,4BAA4B;IAC5C,OAAO,EAAE,2BAA2B,EAAE,CAAC;IACvC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,2BAA2B;IAC3C,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,8BAA8B,EAAE,CAAC;IAC/C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,8BAA8B;IAC9C,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,2BAA2B;IAC3C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,4BAA4B,EAAE,CAAC;IAC5C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,4BAA4B;IAC5C,QAAQ,EAAE,sCAAsC,CAAC;IACjD,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,sCAAsC;IACtD,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,yBAAyB;IACzC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;;;;GAMG;AACH,MAAM,WAAW,0BAA0B;IAC1C,OAAO,EAAE,gCAAgC,EAAE,CAAC;IAC5C,OAAO,EAAE,MAAM,CAAC;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,CAAC,EAAE,+BAA+B,CAAC;IACxC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,gCAAgC;IAChD,KAAK,EAAE,+BAA+B,CAAC;IACvC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,kCAAkC,CAAC;IAC9C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,+BAA+B;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,uCAAuC,EAAE,CAAC;IACvD,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,uCAAuC;IACvD,QAAQ,EAAE,kCAAkC,CAAC;IAC7C,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,kCAAkC;IAClD,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,kCAAkC;IAClD,OAAO,EAAE,iCAAiC,EAAE,CAAC;IAC7C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,iCAAiC;IACjD,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,oCAAoC,EAAE,CAAC;IACrD,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,oCAAoC;IACpD,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,+BAA+B;IAC/C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"}
package/dist/esm/eval.d.ts ADDED
@@ -0,0 +1,36 @@
1
+ /**
2
+ * List of supported Evaluation Frameworks supported in the `eval.yaml` file in benchmarks datasets.
3
+ */
4
+ export declare const EVALUATION_FRAMEWORKS: {
5
+ readonly "inspect-ai": {
6
+ readonly name: "inspect-ai";
7
+ readonly description: "Inspect AI is an open-source framework for large language model evaluations.";
8
+ readonly url: "https://inspect.aisi.org.uk/";
9
+ };
10
+ readonly "math-arena": {
11
+ readonly name: "math-arena";
12
+ readonly description: "MathArena is a platform for evaluation of LLMs on latest math competitions and olympiads.";
13
+ readonly url: "https://github.com/eth-sri/matharena";
14
+ };
15
+ readonly mteb: {
16
+ readonly name: "mteb";
17
+ readonly description: "Multimodal toolbox for evaluating embeddings and retrieval systems.";
18
+ readonly url: "https://github.com/embeddings-benchmark/mteb";
19
+ };
20
+ readonly "olmo-bench": {
21
+ readonly name: "olmo-bench";
22
+ readonly description: "olmOCR-Bench is a framework for evaluating document-level OCR of various tools.";
23
+ readonly url: "https://github.com/allenai/olmocr/tree/main/olmocr/bench";
24
+ };
25
+ readonly harbor: {
26
+ readonly name: "harbor";
27
+ readonly description: "Harbor is a framework for evaluating and optimizing agents and language models.";
28
+ readonly url: "https://github.com/laude-institute/harbor";
29
+ };
30
+ readonly archipelago: {
31
+ readonly name: "archipelago";
32
+ readonly description: "Archipelago is a system for running and evaluating AI agents against MCP applications.";
33
+ readonly url: "https://github.com/Mercor-Intelligence/archipelago";
34
+ };
35
+ };
36
+ //# sourceMappingURL=eval.d.ts.map
package/dist/esm/eval.d.ts.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../src/eval.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+BxB,CAAC"}
package/dist/esm/eval.js ADDED
@@ -0,0 +1,35 @@
1
+ /**
2
+ * List of supported Evaluation Frameworks supported in the `eval.yaml` file in benchmarks datasets.
3
+ */
4
+ export const EVALUATION_FRAMEWORKS = {
5
+ "inspect-ai": {
6
+ name: "inspect-ai",
7
+ description: "Inspect AI is an open-source framework for large language model evaluations.",
8
+ url: "https://inspect.aisi.org.uk/",
9
+ },
10
+ "math-arena": {
11
+ name: "math-arena",
12
+ description: "MathArena is a platform for evaluation of LLMs on latest math competitions and olympiads.",
13
+ url: "https://github.com/eth-sri/matharena",
14
+ },
15
+ mteb: {
16
+ name: "mteb",
17
+ description: "Multimodal toolbox for evaluating embeddings and retrieval systems.",
18
+ url: "https://github.com/embeddings-benchmark/mteb",
19
+ },
20
+ "olmo-bench": {
21
+ name: "olmo-bench",
22
+ description: "olmOCR-Bench is a framework for evaluating document-level OCR of various tools.",
23
+ url: "https://github.com/allenai/olmocr/tree/main/olmocr/bench",
24
+ },
25
+ harbor: {
26
+ name: "harbor",
27
+ description: "Harbor is a framework for evaluating and optimizing agents and language models.",
28
+ url: "https://github.com/laude-institute/harbor",
29
+ },
30
+ archipelago: {
31
+ name: "archipelago",
32
+ description: "Archipelago is a system for running and evaluating AI agents against MCP applications.",
33
+ url: "https://github.com/Mercor-Intelligence/archipelago",
34
+ },
35
+ };
package/dist/esm/index.d.ts CHANGED
@@ -18,4 +18,5 @@ export type { LocalApp, LocalAppKey, LocalAppSnippet } from "./local-apps.js";
18
18
  export { DATASET_LIBRARIES_UI_ELEMENTS } from "./dataset-libraries.js";
19
19
  export type { DatasetLibraryUiElement, DatasetLibraryKey } from "./dataset-libraries.js";
20
20
  export * from "./inference-providers.js";
21
+ export { EVALUATION_FRAMEWORKS } from "./eval.js";
21
22
  //# sourceMappingURL=index.d.ts.map
package/dist/esm/index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AACpE,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACvF,cAAc,kBAAkB,CAAC;AACjC,OAAO,EACN,aAAa,EACb,cAAc,EACd,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,QAAQ,EACb,UAAU,EACV,eAAe,EACf,aAAa,EACb,kBAAkB,GAClB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACN,8BAA8B,EAC9B,sBAAsB,EACtB,2BAA2B,GAC3B,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC9E,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnE,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACzF,YAAY,EACX,aAAa,EACb,sBAAsB,EACtB,gCAAgC,EAChC,8BAA8B,EAC9B,kCAAkC,EAClC,uBAAuB,EACvB,sBAAsB,EACtB,oCAAoC,EACpC,gCAAgC,EAChC,2BAA2B,EAC3B,gCAAgC,EAChC,8BAA8B,EAC9B,sBAAsB,EACtB,8BAA8B,EAC9B,mBAAmB,EACnB,sBAAsB,EACtB,yBAAyB,EACzB,8BAA8B,EAC9B,uBAAuB,GACvB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAC;AAEhE,cAAc,WAAW,CAAC;AAE1B,OAAO,EACN,KAAK,gBAAgB,EACrB,KAAK,wBAAwB,EAC7B,KAAK,gBAAgB,EACrB,yBAAyB,EACzB,yBAAyB,EACzB,iBAAiB,EACjB,oBAAoB,GACpB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,IAAI,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAC7D,YAAY,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAE9E,OAAO,EAAE,6BAA6B,EAAE,MAAM,wBAAwB,CAAC;AACvE,YAAY,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAEzF,cAAc,0BAA0B,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AACpE,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACvF,cAAc,kBAAkB,CAAC;AACjC,OAAO,EACN,aAAa,EACb,cAAc,EACd,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,QAAQ,EACb,UAAU,EACV,eAAe,EACf,aAAa,EACb,kBAAkB,GAClB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACN,8BAA8B,EAC9B,sBAAsB,EACtB,2BAA2B,GAC3B,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC9E,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnE,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACzF,YAAY,EACX,aAAa,EACb,sBAAsB,EACtB,gCAAgC,EAChC,8BAA8B,EAC9B,kCAAkC,EAClC,uBAAuB,EACvB,sBAAsB,EACtB,oCAAoC,EACpC,gCAAgC,EAChC,2BAA2B,EAC3B,gCAAgC,EAChC,8BAA8B,EAC9B,sBAAsB,EACtB,8BAA8B,EAC9B,mBAAmB,EACnB,sBAAsB,EACtB,yBAAyB,EACzB,8BAA8B,EAC9B,uBAAuB,GACvB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAC;AAEhE,cAAc,WAAW,CAAC;AAE1B,OAAO,EACN,KAAK,gBAAgB,EACrB,KAAK,wBAAwB,EAC7B,KAAK,gBAAgB,EACrB,yBAAyB,EACzB,yBAAyB,EACzB,iBAAiB,EACjB,oBAAoB,GACpB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,IAAI,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAC7D,YAAY,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAE9E,OAAO,EAAE,6BAA6B,EAAE,MAAM,wBAAwB,CAAC;AACvE,YAAY,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAEzF,cAAc,0BAA0B,CAAC;AAEzC,OAAO,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC"}
package/dist/esm/index.js CHANGED
@@ -10,3 +10,4 @@ export { SKUS, DEFAULT_MEMORY_OPTIONS } from "./hardware.js";
10
10
  export { LOCAL_APPS } from "./local-apps.js";
11
11
  export { DATASET_LIBRARIES_UI_ELEMENTS } from "./dataset-libraries.js";
12
12
  export * from "./inference-providers.js";
13
+ export { EVALUATION_FRAMEWORKS } from "./eval.js";
package/dist/esm/inference-providers.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- declare const INFERENCE_PROVIDERS: readonly ["cerebras", "cohere", "fal-ai", "fireworks-ai", "hf-inference", "hyperbolic", "ovhcloud", "replicate", "sambanova", "together"];
1
+ declare const INFERENCE_PROVIDERS: readonly ["cerebras", "cohere", "deepinfra", "fal-ai", "fireworks-ai", "hf-inference", "hyperbolic", "ovhcloud", "replicate", "sambanova", "together"];
2
2
  export type SnippetInferenceProvider = (typeof INFERENCE_PROVIDERS)[number] | string;
3
3
  export declare const HF_HUB_INFERENCE_PROXY_TEMPLATE = "https://router.huggingface.co/{{PROVIDER}}";
4
4
  /**
package/dist/esm/inference-providers.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"inference-providers.d.ts","sourceRoot":"","sources":["../../src/inference-providers.ts"],"names":[],"mappings":"AAEA,QAAA,MAAM,mBAAmB,2IAWf,CAAC;AAEX,MAAM,MAAM,wBAAwB,GAAG,CAAC,OAAO,mBAAmB,CAAC,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;AAErF,eAAO,MAAM,+BAA+B,+CAA+C,CAAC;AAE5F;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,wBAAwB,GAAG,MAAM,CAGxE"}
1
+ {"version":3,"file":"inference-providers.d.ts","sourceRoot":"","sources":["../../src/inference-providers.ts"],"names":[],"mappings":"AAEA,QAAA,MAAM,mBAAmB,wJAYf,CAAC;AAEX,MAAM,MAAM,wBAAwB,GAAG,CAAC,OAAO,mBAAmB,CAAC,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;AAErF,eAAO,MAAM,+BAA+B,+CAA+C,CAAC;AAE5F;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,wBAAwB,GAAG,MAAM,CAGxE"}
package/dist/esm/inference-providers.js CHANGED
@@ -3,6 +3,7 @@
3
3
  const INFERENCE_PROVIDERS = [
4
4
  "cerebras",
5
5
  "cohere",
6
+ "deepinfra",
6
7
  "fal-ai",
7
8
  "fireworks-ai",
8
9
  "hf-inference",
package/dist/esm/local-apps.d.ts CHANGED
@@ -212,6 +212,13 @@ export declare const LOCAL_APPS: {
212
212
  displayOnModelPage: (model: ModelData) => boolean;
213
213
  snippet: (model: ModelData, filepath?: string) => LocalAppSnippet[];
214
214
  };
215
+ pi: {
216
+ prettyLabel: string;
217
+ docsUrl: string;
218
+ mainTask: "text-generation";
219
+ displayOnModelPage: (model: ModelData) => boolean;
220
+ snippet: (model: ModelData, filepath?: string) => LocalAppSnippet[];
221
+ };
215
222
  };
216
223
  export type LocalAppKey = keyof typeof LOCAL_APPS;
217
224
  export {};
package/dist/esm/local-apps.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"local-apps.d.ts","sourceRoot":"","sources":["../../src/local-apps.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAKnD,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG;IACtB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,QAAQ,EAAE,YAAY,CAAC;IACvB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;OAEG;IACH,kBAAkB,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,CAAC;CAClD,GAAG,CACD;IACA;;OAEG;IACH,QAAQ,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,GAAG,CAAC;CACtD,GACD;IACA;;;;OAIG;IACH,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM,EAAE,GAAG,eAAe,GAAG,eAAe,EAAE,CAAC;CACzG,CACH,CAAC;AAsBF,iBAAS,UAAU,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAE7C;AAED,iBAAS,mBAAmB,CAAC,KAAK,EAAE,SAAS,WAE5C;AAED,iBAAS,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAU9C;AAED,iBAAS,wBAAwB,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAE3D;AA4UD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,UAAU;;;;;;yBAlUS,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBAiDzC,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBAyC3D,SAAS,KAAG,eAAe,EAAE;;;;;;oCAkQ3B,SAAS;yBA5MT,SAAS,KAAG,eAAe,EAAE;;;;;;;yBAoF9B,SAAS,KAAG,eAAe,EAAE;;;;;;;yBA7B/B,SAAS,KAAG,eAAe,EAAE;;;;;;;;;;;;;;yBApIzB,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yBAJjD,SAAS,aAAa,MAAM,KAAG,MAAM;;;;;;;yBAwM1B,SAAS,aAAa,MAAM,KAAG,MAAM;;;;;;;yBAM9C,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;CAiO5C,CAAC;AAErC,MAAM,MAAM,WAAW,GAAG,MAAM,OAAO,UAAU,CAAC"}
1
+ {"version":3,"file":"local-apps.d.ts","sourceRoot":"","sources":["../../src/local-apps.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAKnD,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG;IACtB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,QAAQ,EAAE,YAAY,CAAC;IACvB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;OAEG;IACH,kBAAkB,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,CAAC;CAClD,GAAG,CACD;IACA;;OAEG;IACH,QAAQ,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,GAAG,CAAC;CACtD,GACD;IACA;;;;OAIG;IACH,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM,EAAE,GAAG,eAAe,GAAG,eAAe,EAAE,CAAC;CACzG,CACH,CAAC;AAsBF,iBAAS,UAAU,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAE7C;AAED,iBAAS,mBAAmB,CAAC,KAAK,EAAE,SAAS,WAE5C;AAED,iBAAS,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAU9C;AAED,iBAAS,wBAAwB,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAE3D;AAiXD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,UAAU;;;;;;yBAvWS,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBAiDzC,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBAyC3D,SAAS,KAAG,eAAe,EAAE;;;;;;oCAuS3B,SAAS;yBAjPT,SAAS,KAAG,eAAe,EAAE;;;;;;;yBAoF9B,SAAS,KAAG,eAAe,EAAE;;;;;;;yBA7B/B,SAAS,KAAG,eAAe,EAAE;;;;;;;;;;;;;;yBApIzB,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yBAJjD,SAAS,aAAa,MAAM,KAAG,MAAM;;;;;;;yBA6O1B,SAAS,aAAa,MAAM,KAAG,MAAM;;;;;;;yBAM9C,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBA3CtD,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;CAmRtC,CAAC;AAErC,MAAM,MAAM,WAAW,GAAG,MAAM,OAAO,UAAU,CAAC"}
package/dist/esm/local-apps.js CHANGED
@@ -294,6 +294,36 @@ const snippetMlxLm = (model) => {
294
294
  : []),
295
295
  ];
296
296
  };
297
+ const snippetPi = (model, filepath) => {
298
+ const quantTag = getQuantTag(filepath);
299
+ const modelName = model.id.split("/").pop() ?? model.id;
300
+ const modelsJson = JSON.stringify({
301
+ providers: {
302
+ "llama-cpp": {
303
+ baseUrl: "http://localhost:8080/v1",
304
+ api: "openai-completions",
305
+ apiKey: "none",
306
+ models: [{ id: modelName }],
307
+ },
308
+ },
309
+ }, null, 2);
310
+ return [
311
+ {
312
+ title: "Start the llama.cpp server",
313
+ setup: "# Install llama.cpp:\nbrew install llama.cpp",
314
+ content: `# Start a local OpenAI-compatible server:\nllama-server -hf ${model.id}${quantTag} --jinja`,
315
+ },
316
+ {
317
+ title: "Configure the model in Pi",
318
+ setup: "# Install Pi:\nnpm install -g @mariozechner/pi-coding-agent",
319
+ content: `# Add to ~/.pi/agent/models.json:\n${modelsJson}`,
320
+ },
321
+ {
322
+ title: "Run Pi",
323
+ content: `# Start Pi in your project directory:\npi`,
324
+ },
325
+ ];
326
+ };
297
327
  const snippetDockerModelRunner = (model, filepath) => {
298
328
  // Only add quant tag for GGUF models, not safetensors
299
329
  const quantTag = isLlamaCppGgufModel(model) ? getQuantTag(filepath) : "";
@@ -520,4 +550,11 @@ export const LOCAL_APPS = {
520
550
  displayOnModelPage: (model) => isLlamaCppGgufModel(model) || isAmdRyzenModel(model),
521
551
  snippet: snippetLemonade,
522
552
  },
553
+ pi: {
554
+ prettyLabel: "Pi",
555
+ docsUrl: "https://github.com/badlogic/pi-mono",
556
+ mainTask: "text-generation",
557
+ displayOnModelPage: (model) => isLlamaCppGgufModel(model) && !!model.gguf?.chat_template?.includes("tools"),
558
+ snippet: snippetPi,
559
+ },
523
560
  };
package/dist/esm/local-apps.spec.js CHANGED
@@ -110,6 +110,20 @@ curl -X POST "http://localhost:8000/v1/chat/completions" \\
110
110
  ]
111
111
  }'`);
112
112
  });
113
+ it("pi", async () => {
114
+ const { snippet: snippetFunc } = LOCAL_APPS["pi"];
115
+ const model = {
116
+ id: "bartowski/Llama-3.2-3B-Instruct-GGUF",
117
+ tags: ["conversational"],
118
+ gguf: { total: 1, context_length: 4096, chat_template: "{% if tools %}" },
119
+ inference: "",
120
+ };
121
+ const snippet = snippetFunc(model);
122
+ expect(snippet[0].content).toContain(`llama-server -hf bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}} --jinja`);
123
+ expect(snippet[1].setup).toContain("npm install -g @mariozechner/pi-coding-agent");
124
+ expect(snippet[1].content).toContain(`"id": "Llama-3.2-3B-Instruct-GGUF"`);
125
+ expect(snippet[2].content).toContain("pi");
126
+ });
113
127
  it("docker model runner", async () => {
114
128
  const { snippet: snippetFunc } = LOCAL_APPS["docker-model-runner"];
115
129
  const model = {
package/dist/esm/model-data.d.ts CHANGED
@@ -127,6 +127,7 @@ export interface ModelData {
127
127
  total: number;
128
128
  architecture?: string;
129
129
  context_length?: number;
130
+ chat_template?: string;
130
131
  };
131
132
  }
132
133
  /**
package/dist/esm/model-data.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"model-data.d.ts","sourceRoot":"","sources":["../../src/model-data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAE3D;;GAEG;AACH,MAAM,WAAW,SAAS;IACzB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IACX;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;OAEG;IACH,MAAM,CAAC,EAAE;QACR,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;QACzB;;WAEG;QACH,QAAQ,CAAC,EAAE;YACV;;eAEG;YACH,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;SACpB,CAAC;QACF,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,mBAAmB,CAAC,EAAE;YACrB,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,YAAY,CAAC,EAAE,OAAO,CAAC;YACvB,YAAY,CAAC,EAAE,OAAO,CAAC;YACvB;;eAEG;YACH,YAAY,CAAC,EAAE,MAAM,CAAC;SACtB,CAAC;QACF,gBAAgB,CAAC,EAAE,eAAe,CAAC;QACnC,gBAAgB,CAAC,EAAE;YAClB,aAAa,CAAC,EAAE,MAAM,CAAC;SACvB,CAAC;QACF,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,oBAAoB,CAAC,EAAE;YACtB,UAAU,CAAC,EAAE,MAAM,CAAC;YACpB,WAAW,CAAC,EAAE,MAAM,CAAC;SACrB,CAAC;QACF,SAAS,CAAC,EAAE;YACX,WAAW,CAAC,EAAE,MAAM,CAAC;SACrB,CAAC;QACF,OAAO,CAAC,EAAE;YACT,KAAK,CAAC,EAAE;gBACP,IAAI,CAAC,EAAE,MAAM,CAAC;aACd,CAAC;YACF,YAAY,CAAC,EAAE,MAAM,CAAC;SACtB,CAAC;QACF,WAAW,CAAC,EAAE;YACb,qBAAqB,CAAC,EAAE,MAAM,CAAC;YAC/B,iBAAiB,CAAC,EAAE,MAAM,CAAC;YAC3B,gBAAgB,CAAC,EAAE,MAAM,CAAC;SAC1B,CAAC;QACF,IAAI,CAAC,EAAE;YACN,uBAAuB,CAAC,EAAE,MAAM,CAAC;YACjC,SAAS,CAAC,EAAE,MAAM,CAAC;SACnB,CAAC;QACF,SAAS,CAAC,EAAE;YACX,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;SACjB,CAAC;KACF,CAAC;IACF;;OAEG;IACH,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;;OAEG;IACH,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC;;OAEG;IACH,YAAY,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;IACxC;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC;;;;;OAKG;IACH,UAAU,CAAC,EAAE,aAAa,EAAE,GAAG,SAAS,CAAC;IACzC;;;;;;;;;OASG;IACH,QAAQ,CAAC,EAAE;QACV,SAAS,CAAC,EACP,OAAO,GACP;YACA,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;SACpC,CAAC;QACL,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;QAC/B,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;KAChC,CAAC;IACF;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE;QACb,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACnC,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,OAAO,CAAC;KACjB,CAAC;IACF,IAAI,CAAC,EAAE;QACN,KAAK,EAAE,MAAM,CAAC;QACd,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,cAAc,CAAC,EAAE,MAAM,CAAC;KACxB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB"}
1
+ {"version":3,"file":"model-data.d.ts","sourceRoot":"","sources":["../../src/model-data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAE3D;;GAEG;AACH,MAAM,WAAW,SAAS;IACzB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IACX;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;OAEG;IACH,MAAM,CAAC,EAAE;QACR,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;QACzB;;WAEG;QACH,QAAQ,CAAC,EAAE;YACV;;eAEG;YACH,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;SACpB,CAAC;QACF,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,mBAAmB,CAAC,EAAE;YACrB,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,YAAY,CAAC,EAAE,OAAO,CAAC;YACvB,YAAY,CAAC,EAAE,OAAO,CAAC;YACvB;;eAEG;YACH,YAAY,CAAC,EAAE,MAAM,CAAC;SACtB,CAAC;QACF,gBAAgB,CAAC,EAAE,eAAe,CAAC;QACnC,gBAAgB,CAAC,EAAE;YAClB,aAAa,CAAC,EAAE,MAAM,CAAC;SACvB,CAAC;QACF,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,oBAAoB,CAAC,EAAE;YACtB,UAAU,CAAC,EAAE,MAAM,CAAC;YACpB,WAAW,CAAC,EAAE,MAAM,CAAC;SACrB,CAAC;QACF,SAAS,CAAC,EAAE;YACX,WAAW,CAAC,EAAE,MAAM,CAAC;SACrB,CAAC;QACF,OAAO,CAAC,EAAE;YACT,KAAK,CAAC,EAAE;gBACP,IAAI,CAAC,EAAE,MAAM,CAAC;aACd,CAAC;YACF,YAAY,CAAC,EAAE,MAAM,CAAC;SACtB,CAAC;QACF,WAAW,CAAC,EAAE;YACb,qBAAqB,CAAC,EAAE,MAAM,CAAC;YAC/B,iBAAiB,CAAC,EAAE,MAAM,CAAC;YAC3B,gBAAgB,CAAC,EAAE,MAAM,CAAC;SAC1B,CAAC;QACF,IAAI,CAAC,EAAE;YACN,uBAAuB,CAAC,EAAE,MAAM,CAAC;YACjC,SAAS,CAAC,EAAE,MAAM,CAAC;SACnB,CAAC;QACF,SAAS,CAAC,EAAE;YACX,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;SACjB,CAAC;KACF,CAAC;IACF;;OAEG;IACH,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;;OAEG;IACH,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC;;OAEG;IACH,YAAY,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;IACxC;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC;;;;;OAKG;IACH,UAAU,CAAC,EAAE,aAAa,EAAE,GAAG,SAAS,CAAC;IACzC;;;;;;;;;OASG;IACH,QAAQ,CAAC,EAAE;QACV,SAAS,CAAC,EACP,OAAO,GACP;YACA,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;SACpC,CAAC;QACL,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;QAC/B,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;KAChC,CAAC;IACF;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE;QACb,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACnC,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,OAAO,CAAC;KACjB,CAAC;IACF,IAAI,CAAC,EAAE;QACN,KAAK,EAAE,MAAM,CAAC;QACd,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,aAAa,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB"}
package/dist/esm/tasks/chat-completion/inference.d.ts CHANGED
@@ -62,6 +62,10 @@ export interface ChatCompletionInput {
62
62
  * increasing the model's likelihood to talk about new topics
63
63
  */
64
64
  presence_penalty?: number;
65
+ /**
66
+ * Optional. Constrains effort on reasoning for reasoning models. Reducing reasoning effort can result in faster responses and fewer tokens used on reasoning. Common values: none, minimal, low, medium, high, xhigh. Support and defaults are provider and model-dependent.
67
+ */
68
+ reasoning_effort?: string;
65
69
  response_format?: ChatCompletionInputGrammarType;
66
70
  seed?: number;
67
71
  /**
package/dist/esm/tasks/chat-completion/inference.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/chat-completion/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;;;;;GAMG;AACH,MAAM,WAAW,mBAAmB;IACnC;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;;;;;;;;;OAWG;IACH,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,QAAQ,EAAE,0BAA0B,EAAE,CAAC;IACvC;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;OAKG;IACH,CAAC,CAAC,EAAE,MAAM,CAAC;IACX;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,8BAA8B,CAAC;IACjD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,cAAc,CAAC,EAAE,gCAAgC,CAAC;IAClD;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,6BAA6B,CAAC;IAC5C;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,KAAK,CAAC,EAAE,uBAAuB,EAAE,CAAC;IAClC;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;OAKG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,0BAA0B;IAC1C,OAAO,CAAC,EAAE,iCAAiC,CAAC;IAC5C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,2BAA2B,EAAE,CAAC;IAC3C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,iCAAiC,GAAG,+BAA+B,EAAE,GAAG,MAAM,CAAC;AAC3F,MAAM,WAAW,+BAA+B;IAC/C,SAAS,CAAC,EAAE,sBAAsB,CAAC;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,mCAAmC,CAAC;IAC1C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,sBAAsB;IACtC,GAAG,EAAE,MAAM,CAAC;IACZ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,mCAAmC,GAAG,MAAM,GAAG,WAAW,CAAC;AACvE,MAAM,WAAW,2BAA2B;IAC3C,QAAQ,EAAE,qCAAqC,CAAC;IAChD,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,qCAAqC;IACrD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,8BAA8B;IAC9C,WAAW,CAAC,EAAE,mCAAmC,CAAC;IAClD,IAAI,EAAE,kCAAkC,CAAC;IACzC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,mCAAmC;IACnD;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,MAAM,CAAC,EAAE;QACR,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACvB,CAAC;IACF;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,kCAAkC,GAAG,MAAM,GAAG,aAAa,GAAG,aAAa,CAAC;AACxF,MAAM,WAAW,gCAAgC;IAChD;;;;;OAKG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;GAGG;AACH,MAAM,MAAM,6BAA6B,GAAG,iCAAiC,GAAG,mCAAmC,CAAC;AACpH;;;;;;GAMG;AACH,MAAM,MAAM,iCAAiC,GAAG,MAAM,GAAG,MAAM,GAAG,UAAU,CAAC;AAC7E,MAAM,WAAW,mCAAmC;IACnD,QAAQ,EAAE,+BAA+B,CAAC;IAC1C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,+BAA+B;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,uBAAuB;IACvC,QAAQ,EAAE,qCAAqC,CAAC;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB;IACpC,OAAO,EAAE,4BAA4B,EAAE,CAAC;IACxC,OAAO,EAAE,MAAM,CAAC;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,EAAE,yBAAyB,CAAC;IACjC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,4BAA4B;IAC5C,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,4BAA4B,CAAC;IACxC,OAAO,EAAE,2BAA2B,CAAC;IACrC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,4BAA4B;IAC5C,OAAO,EAAE,2BAA2B,EAAE,CAAC;IACvC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,2BAA2B;IAC3C,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,8BAA8B,EAAE,CAAC;IAC/C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,8BAA8B;IAC9C,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,2BAA2B;IAC3C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,4BAA4B,EAAE,CAAC;IAC5C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,4BAA4B;IAC5C,QAAQ,EAAE,sCAAsC,CAAC;IACjD,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,sCAAsC;IACtD,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,yBAAyB;IACzC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;;;;GAMG;AACH,MAAM,WAAW,0BAA0B;IAC1C,OAAO,EAAE,gCAAgC,EAAE,CAAC;IAC5C,OAAO,EAAE,MAAM,CAAC;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,CAAC,EAAE,+BAA+B,CAAC;IACxC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,gCAAgC;IAChD,KAAK,EAAE,+BAA+B,CAAC;IACvC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,kCAAkC,CAAC;IAC9C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,+BAA+B;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,uCAAuC,EAAE,CAAC;IACvD,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,uCAAuC;IACvD,QAAQ,EAAE,kCAAkC,CAAC;IAC7C,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,kCAAkC;IAClD,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,kCAAkC;IAClD,OAAO,EAAE,iCAAiC,EAAE,CAAC;IAC7C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,iCAAiC;IACjD,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,oCAAoC,EAAE,CAAC;IACrD,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,oCAAoC;IACpD,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,+BAA+B;IAC/C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"}
1
+ {"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/chat-completion/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;;;;;GAMG;AACH,MAAM,WAAW,mBAAmB;IACnC;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;;;;;;;;;OAWG;IACH,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,QAAQ,EAAE,0BAA0B,EAAE,CAAC;IACvC;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;OAKG;IACH,CAAC,CAAC,EAAE,MAAM,CAAC;IACX;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,8BAA8B,CAAC;IACjD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,cAAc,CAAC,EAAE,gCAAgC,CAAC;IAClD;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,6BAA6B,CAAC;IAC5C;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,KAAK,CAAC,EAAE,uBAAuB,EAAE,CAAC;IAClC;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;OAKG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,0BAA0B;IAC1C,OAAO,CAAC,EAAE,iCAAiC,CAAC;IAC5C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,2BAA2B,EAAE,CAAC;IAC3C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,iCAAiC,GAAG,+BAA+B,EAAE,GAAG,MAAM,CAAC;AAC3F,MAAM,WAAW,+BAA+B;IAC/C,SAAS,CAAC,EAAE,sBAAsB,CAAC;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,mCAAmC,CAAC;IAC1C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,sBAAsB;IACtC,GAAG,EAAE,MAAM,CAAC;IACZ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,mCAAmC,GAAG,MAAM,GAAG,WAAW,CAAC;AACvE,MAAM,WAAW,2BAA2B;IAC3C,QAAQ,EAAE,qCAAqC,CAAC;IAChD,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,qCAAqC;IACrD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,8BAA8B;IAC9C,WAAW,CAAC,EAAE,mCAAmC,CAAC;IAClD,IAAI,EAAE,kCAAkC,CAAC;IACzC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,mCAAmC;IACnD;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,MAAM,CAAC,EAAE;QACR,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACvB,CAAC;IACF;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,kCAAkC,GAAG,MAAM,GAAG,aAAa,GAAG,aAAa,CAAC;AACxF,MAAM,WAAW,gCAAgC;IAChD;;;;;OAKG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;GAGG;AACH,MAAM,MAAM,6BAA6B,GAAG,iCAAiC,GAAG,mCAAmC,CAAC;AACpH;;;;;;GAMG;AACH,MAAM,MAAM,iCAAiC,GAAG,MAAM,GAAG,MAAM,GAAG,UAAU,CAAC;AAC7E,MAAM,WAAW,mCAAmC;IACnD,QAAQ,EAAE,+BAA+B,CAAC;IAC1C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,+BAA+B;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,uBAAuB;IACvC,QAAQ,EAAE,qCAAqC,CAAC;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB;IACpC,OAAO,EAAE,4BAA4B,EAAE,CAAC;IACxC,OAAO,EAAE,MAAM,CAAC;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,EAAE,yBAAyB,CAAC;IACjC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,4BAA4B;IAC5C,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,4BAA4B,CAAC;IACxC,OAAO,EAAE,2BAA2B,CAAC;IACrC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,4BAA4B;IAC5C,OAAO,EAAE,2BAA2B,EAAE,CAAC;IACvC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,2BAA2B;IAC3C,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,8BAA8B,EAAE,CAAC;IAC/C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,8BAA8B;IAC9C,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,2BAA2B;IAC3C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,4BAA4B,EAAE,CAAC;IAC5C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,4BAA4B;IAC5C,QAAQ,EAAE,sCAAsC,CAAC;IACjD,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,sCAAsC;IACtD,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,yBAAyB;IACzC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;;;;GAMG;AACH,MAAM,WAAW,0BAA0B;IAC1C,OAAO,EAAE,gCAAgC,EAAE,CAAC;IAC5C,OAAO,EAAE,MAAM,CAAC;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,CAAC,EAAE,+BAA+B,CAAC;IACxC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,gCAAgC;IAChD,KAAK,EAAE,+BAA+B,CAAC;IACvC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,kCAAkC,CAAC;IAC9C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,+BAA+B;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,uCAAuC,EAAE,CAAC;IACvD,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,uCAAuC;IACvD,QAAQ,EAAE,kCAAkC,CAAC;IAC7C,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,kCAAkC;IAClD,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,kCAAkC;IAClD,OAAO,EAAE,iCAAiC,EAAE,CAAC;IAC7C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,iCAAiC;IACjD,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,oCAAoC,EAAE,CAAC;IACrD,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,oCAAoC;IACpD,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,+BAA+B;IAC/C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@huggingface/tasks",
3
- "version": "0.19.85",
3
+ "version": "0.19.86",
4
4
  "description": "List of ML tasks for huggingface.co/tasks",
5
5
  "keywords": [
6
6
  "hub",
package/src/eval.ts ADDED
@@ -0,0 +1,35 @@
1
+ /**
2
+ * List of supported Evaluation Frameworks supported in the `eval.yaml` file in benchmarks datasets.
3
+ */
4
+ export const EVALUATION_FRAMEWORKS = {
5
+ "inspect-ai": {
6
+ name: "inspect-ai",
7
+ description: "Inspect AI is an open-source framework for large language model evaluations.",
8
+ url: "https://inspect.aisi.org.uk/",
9
+ },
10
+ "math-arena": {
11
+ name: "math-arena",
12
+ description: "MathArena is a platform for evaluation of LLMs on latest math competitions and olympiads.",
13
+ url: "https://github.com/eth-sri/matharena",
14
+ },
15
+ mteb: {
16
+ name: "mteb",
17
+ description: "Multimodal toolbox for evaluating embeddings and retrieval systems.",
18
+ url: "https://github.com/embeddings-benchmark/mteb",
19
+ },
20
+ "olmo-bench": {
21
+ name: "olmo-bench",
22
+ description: "olmOCR-Bench is a framework for evaluating document-level OCR of various tools.",
23
+ url: "https://github.com/allenai/olmocr/tree/main/olmocr/bench",
24
+ },
25
+ harbor: {
26
+ name: "harbor",
27
+ description: "Harbor is a framework for evaluating and optimizing agents and language models.",
28
+ url: "https://github.com/laude-institute/harbor",
29
+ },
30
+ archipelago: {
31
+ name: "archipelago",
32
+ description: "Archipelago is a system for running and evaluating AI agents against MCP applications.",
33
+ url: "https://github.com/Mercor-Intelligence/archipelago",
34
+ },
35
+ } as const;
package/src/index.ts CHANGED
@@ -66,3 +66,5 @@ export { DATASET_LIBRARIES_UI_ELEMENTS } from "./dataset-libraries.js";
66
66
  export type { DatasetLibraryUiElement, DatasetLibraryKey } from "./dataset-libraries.js";
67
67
 
68
68
  export * from "./inference-providers.js";
69
+
70
+ export { EVALUATION_FRAMEWORKS } from "./eval.js";
package/src/inference-providers.ts CHANGED
@@ -3,6 +3,7 @@
3
3
  const INFERENCE_PROVIDERS = [
4
4
  "cerebras",
5
5
  "cohere",
6
+ "deepinfra",
6
7
  "fal-ai",
7
8
  "fireworks-ai",
8
9
  "hf-inference",
package/src/local-apps.spec.ts CHANGED
@@ -122,6 +122,22 @@ curl -X POST "http://localhost:8000/v1/chat/completions" \\
122
122
  }'`);
123
123
  });
124
124
 
125
+ it("pi", async () => {
126
+ const { snippet: snippetFunc } = LOCAL_APPS["pi"];
127
+ const model: ModelData = {
128
+ id: "bartowski/Llama-3.2-3B-Instruct-GGUF",
129
+ tags: ["conversational"],
130
+ gguf: { total: 1, context_length: 4096, chat_template: "{% if tools %}" },
131
+ inference: "",
132
+ };
133
+ const snippet = snippetFunc(model);
134
+
135
+ expect(snippet[0].content).toContain(`llama-server -hf bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}} --jinja`);
136
+ expect(snippet[1].setup).toContain("npm install -g @mariozechner/pi-coding-agent");
137
+ expect(snippet[1].content).toContain(`"id": "Llama-3.2-3B-Instruct-GGUF"`);
138
+ expect(snippet[2].content).toContain("pi");
139
+ });
140
+
125
141
  it("docker model runner", async () => {
126
142
  const { snippet: snippetFunc } = LOCAL_APPS["docker-model-runner"];
127
143
  const model: ModelData = {
package/src/local-apps.ts CHANGED
@@ -389,6 +389,43 @@ const snippetMlxLm = (model: ModelData): LocalAppSnippet[] => {
389
389
  ];
390
390
  };
391
391
 
392
+ const snippetPi = (model: ModelData, filepath?: string): LocalAppSnippet[] => {
393
+ const quantTag = getQuantTag(filepath);
394
+ const modelName = model.id.split("/").pop() ?? model.id;
395
+
396
+ const modelsJson = JSON.stringify(
397
+ {
398
+ providers: {
399
+ "llama-cpp": {
400
+ baseUrl: "http://localhost:8080/v1",
401
+ api: "openai-completions",
402
+ apiKey: "none",
403
+ models: [{ id: modelName }],
404
+ },
405
+ },
406
+ },
407
+ null,
408
+ 2,
409
+ );
410
+
411
+ return [
412
+ {
413
+ title: "Start the llama.cpp server",
414
+ setup: "# Install llama.cpp:\nbrew install llama.cpp",
415
+ content: `# Start a local OpenAI-compatible server:\nllama-server -hf ${model.id}${quantTag} --jinja`,
416
+ },
417
+ {
418
+ title: "Configure the model in Pi",
419
+ setup: "# Install Pi:\nnpm install -g @mariozechner/pi-coding-agent",
420
+ content: `# Add to ~/.pi/agent/models.json:\n${modelsJson}`,
421
+ },
422
+ {
423
+ title: "Run Pi",
424
+ content: `# Start Pi in your project directory:\npi`,
425
+ },
426
+ ];
427
+ };
428
+
392
429
  const snippetDockerModelRunner = (model: ModelData, filepath?: string): string => {
393
430
  // Only add quant tag for GGUF models, not safetensors
394
431
  const quantTag = isLlamaCppGgufModel(model) ? getQuantTag(filepath) : "";
@@ -620,6 +657,13 @@ export const LOCAL_APPS = {
620
657
  displayOnModelPage: (model) => isLlamaCppGgufModel(model) || isAmdRyzenModel(model),
621
658
  snippet: snippetLemonade,
622
659
  },
660
+ pi: {
661
+ prettyLabel: "Pi",
662
+ docsUrl: "https://github.com/badlogic/pi-mono",
663
+ mainTask: "text-generation",
664
+ displayOnModelPage: (model) => isLlamaCppGgufModel(model) && !!model.gguf?.chat_template?.includes("tools"),
665
+ snippet: snippetPi,
666
+ },
623
667
  } satisfies Record<string, LocalApp>;
624
668
 
625
669
  export type LocalAppKey = keyof typeof LOCAL_APPS;
package/src/model-data.ts CHANGED
@@ -130,6 +130,7 @@ export interface ModelData {
130
130
  total: number;
131
131
  architecture?: string;
132
132
  context_length?: number;
133
+ chat_template?: string;
133
134
  };
134
135
  }
135
136
 
package/src/tasks/chat-completion/inference.ts CHANGED
@@ -62,6 +62,10 @@ export interface ChatCompletionInput {
62
62
  * increasing the model's likelihood to talk about new topics
63
63
  */
64
64
  presence_penalty?: number;
65
+ /**
66
+ * Optional. Constrains effort on reasoning for reasoning models. Reducing reasoning effort can result in faster responses and fewer tokens used on reasoning. Common values: none, minimal, low, medium, high, xhigh. Support and defaults are provider and model-dependent.
67
+ */
68
+ reasoning_effort?: string;
65
69
  response_format?: ChatCompletionInputGrammarType;
66
70
  seed?: number;
67
71
  /**
package/src/tasks/chat-completion/spec/input.json CHANGED
@@ -66,6 +66,11 @@
66
66
  "example": 0.1,
67
67
  "nullable": true
68
68
  },
69
+ "reasoning_effort": {
70
+ "type": "string",
71
+ "description": "Optional. Constrains effort on reasoning for models that support reasoning. Reducing reasoning effort can result in faster responses and fewer tokens used on reasoning. Common values: none, minimal, low, medium, high, xhigh. Support and defaults are provider and model-dependent.",
72
+ "nullable": true
73
+ },
69
74
  "response_format": {
70
75
  "allOf": [
71
76
  {