@arizeai/phoenix-evals 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +205 -0
- package/dist/esm/llm/ClassificationEvaluator.d.ts.map +1 -1
- package/dist/esm/llm/ClassificationEvaluator.js +1 -1
- package/dist/esm/llm/ClassificationEvaluator.js.map +1 -1
- package/dist/esm/llm/LLMEvaluator.d.ts +2 -2
- package/dist/esm/llm/LLMEvaluator.d.ts.map +1 -1
- package/dist/esm/llm/LLMEvaluator.js +1 -1
- package/dist/esm/llm/LLMEvaluator.js.map +1 -1
- package/dist/esm/llm/createClassificationEvaluator.d.ts.map +1 -1
- package/dist/esm/llm/createClassificationEvaluator.js.map +1 -1
- package/dist/esm/llm/createClassifierFn.d.ts.map +1 -1
- package/dist/esm/llm/createClassifierFn.js +1 -1
- package/dist/esm/llm/createClassifierFn.js.map +1 -1
- package/dist/esm/llm/createDocumentRelevancyEvaluator.d.ts.map +1 -1
- package/dist/esm/llm/createDocumentRelevancyEvaluator.js +1 -1
- package/dist/esm/llm/createDocumentRelevancyEvaluator.js.map +1 -1
- package/dist/esm/llm/createHallucinationEvaluator.d.ts.map +1 -1
- package/dist/esm/llm/createHallucinationEvaluator.js +1 -1
- package/dist/esm/llm/createHallucinationEvaluator.js.map +1 -1
- package/dist/esm/llm/generateClassification.d.ts.map +1 -1
- package/dist/esm/llm/generateClassification.js +1 -1
- package/dist/esm/llm/generateClassification.js.map +1 -1
- package/dist/esm/template/applyTemplate.d.ts.map +1 -1
- package/dist/esm/template/applyTemplate.js.map +1 -1
- package/dist/esm/template/getTemplateVariables.d.ts.map +1 -1
- package/dist/esm/template/getTemplateVariables.js.map +1 -1
- package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
- package/dist/esm/types/evals.d.ts +5 -5
- package/dist/esm/types/evals.d.ts.map +1 -1
- package/dist/src/llm/ClassificationEvaluator.d.ts.map +1 -1
- package/dist/src/llm/ClassificationEvaluator.js +1 -1
- package/dist/src/llm/ClassificationEvaluator.js.map +1 -1
- package/dist/src/llm/LLMEvaluator.d.ts +2 -2
- package/dist/src/llm/LLMEvaluator.d.ts.map +1 -1
- package/dist/src/llm/LLMEvaluator.js +1 -1
- package/dist/src/llm/LLMEvaluator.js.map +1 -1
- package/dist/src/llm/createClassificationEvaluator.d.ts.map +1 -1
- package/dist/src/llm/createClassificationEvaluator.js.map +1 -1
- package/dist/src/llm/createClassifierFn.d.ts.map +1 -1
- package/dist/src/llm/createClassifierFn.js +1 -1
- package/dist/src/llm/createClassifierFn.js.map +1 -1
- package/dist/src/llm/createDocumentRelevancyEvaluator.d.ts.map +1 -1
- package/dist/src/llm/createDocumentRelevancyEvaluator.js.map +1 -1
- package/dist/src/llm/createHallucinationEvaluator.d.ts.map +1 -1
- package/dist/src/llm/createHallucinationEvaluator.js.map +1 -1
- package/dist/src/llm/generateClassification.d.ts.map +1 -1
- package/dist/src/llm/generateClassification.js +1 -1
- package/dist/src/llm/generateClassification.js.map +1 -1
- package/dist/src/template/applyTemplate.d.ts.map +1 -1
- package/dist/src/template/applyTemplate.js.map +1 -1
- package/dist/src/template/getTemplateVariables.d.ts.map +1 -1
- package/dist/src/template/getTemplateVariables.js.map +1 -1
- package/dist/src/types/evals.d.ts +5 -5
- package/dist/src/types/evals.d.ts.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +5 -3
- package/src/llm/ClassificationEvaluator.ts +2 -1
- package/src/llm/LLMEvaluator.ts +2 -2
- package/src/llm/createClassificationEvaluator.ts +1 -0
- package/src/llm/createClassifierFn.ts +3 -2
- package/src/llm/createDocumentRelevancyEvaluator.ts +3 -2
- package/src/llm/createHallucinationEvaluator.ts +4 -3
- package/src/llm/generateClassification.ts +2 -1
- package/src/template/applyTemplate.ts +1 -0
- package/src/template/getTemplateVariables.ts +1 -0
- package/src/types/evals.ts +6 -5
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { EvaluationResult, Evaluator, OptimizationDirection
|
|
1
|
+
import { CreateEvaluatorArgs, EvaluationResult, Evaluator, OptimizationDirection } from "../types";
|
|
2
2
|
/**
|
|
3
3
|
* Base class for llm evaluation metrics / scores
|
|
4
4
|
*/
|
|
5
5
|
export declare abstract class LLMEvaluator<RecordType extends Record<string, unknown>> implements Evaluator<RecordType> {
|
|
6
6
|
readonly name: string;
|
|
7
|
-
readonly
|
|
7
|
+
readonly kind: "LLM";
|
|
8
8
|
readonly optimizationDirection?: OptimizationDirection;
|
|
9
9
|
constructor({ name, optimizationDirection }: CreateEvaluatorArgs);
|
|
10
10
|
abstract evaluate(_example: RecordType): Promise<EvaluationResult>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LLMEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/LLMEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,gBAAgB,EAChB,SAAS,EACT,qBAAqB,
|
|
1
|
+
{"version":3,"file":"LLMEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/LLMEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,mBAAmB,EACnB,gBAAgB,EAChB,SAAS,EACT,qBAAqB,EACtB,MAAM,UAAU,CAAC;AAElB;;GAEG;AACH,8BAAsB,YAAY,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC3E,YAAW,SAAS,CAAC,UAAU,CAAC;IAEhC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAG,KAAK,CAAU;IAC/B,QAAQ,CAAC,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;gBAC3C,EAAE,IAAI,EAAE,qBAAqB,EAAE,EAAE,mBAAmB;IAIhE,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;CACnE"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LLMEvaluator.js","sourceRoot":"","sources":["../../../src/llm/LLMEvaluator.ts"],"names":[],"mappings":";;;AAOA;;GAEG;AACH,MAAsB,YAAY;IAMhC,YAAY,EAAE,IAAI,EAAE,qBAAqB,EAAuB;QAFvD,
|
|
1
|
+
{"version":3,"file":"LLMEvaluator.js","sourceRoot":"","sources":["../../../src/llm/LLMEvaluator.ts"],"names":[],"mappings":";;;AAOA;;GAEG;AACH,MAAsB,YAAY;IAMhC,YAAY,EAAE,IAAI,EAAE,qBAAqB,EAAuB;QAFvD,SAAI,GAAG,KAAc,CAAC;QAG7B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;IACrD,CAAC;CAEF;AAXD,oCAWC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"createClassificationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createClassificationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"createClassificationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createClassificationEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AAEnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,wBAAgB,6BAA6B,CAC3C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAE1C,IAAI,EAAE,iCAAiC,GACtC,uBAAuB,CAAC,UAAU,CAAC,CAErC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"createClassificationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createClassificationEvaluator.ts"],"names":[],"mappings":";;
|
|
1
|
+
{"version":3,"file":"createClassificationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createClassificationEvaluator.ts"],"names":[],"mappings":";;AAIA,sEAMC;AARD,uEAAoE;AAEpE,SAAgB,6BAA6B,CAG3C,IAAuC;IAEvC,OAAO,IAAI,iDAAuB,CAAa,IAAI,CAAC,CAAC;AACvD,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"createClassifierFn.d.ts","sourceRoot":"","sources":["../../../src/llm/createClassifierFn.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"createClassifierFn.d.ts","sourceRoot":"","sources":["../../../src/llm/createClassifierFn.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,oBAAoB,EAEpB,WAAW,EACZ,MAAM,gBAAgB,CAAC;AAkBxB;;GAEG;AACH,wBAAgB,kBAAkB,CAChC,gBAAgB,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChD,IAAI,EAAE,oBAAoB,GAAG,WAAW,CAAC,gBAAgB,CAAC,CA4B3D"}
|
|
@@ -12,8 +12,8 @@ var __rest = (this && this.__rest) || function (s, e) {
|
|
|
12
12
|
};
|
|
13
13
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
14
|
exports.createClassifierFn = createClassifierFn;
|
|
15
|
-
const generateClassification_1 = require("./generateClassification");
|
|
16
15
|
const template_1 = require("../template");
|
|
16
|
+
const generateClassification_1 = require("./generateClassification");
|
|
17
17
|
/**
|
|
18
18
|
* Convert a mapping of choices to labels
|
|
19
19
|
* Asserts that the choices are valid
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"createClassifierFn.js","sourceRoot":"","sources":["../../../src/llm/createClassifierFn.ts"],"names":[],"mappings":";;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"createClassifierFn.js","sourceRoot":"","sources":["../../../src/llm/createClassifierFn.ts"],"names":[],"mappings":";;;;;;;;;;;;;AA2BA,gDA8BC;AAzDD,0CAA6C;AAQ7C,qEAAkE;AAElE;;;GAGG;AACH,SAAS,eAAe,CACtB,OAAiC;IAEjC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,MAA+B,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,SAAgB,kBAAkB,CAEhC,IAA0B;IAC1B,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,cAAc,KAAc,IAAI,EAAb,IAAI,UAAK,IAAI,EAAlD,sCAA2C,CAAO,CAAC;IAEzD,OAAO,KAAK,EAAE,IAAsB,EAA6B,EAAE;QACjE,MAAM,iBAAiB,qBAClB,IAAI,CACR,CAAC;QAEF,MAAM,MAAM,GAAG,IAAA,yBAAc,EAAC;YAC5B,QAAQ,EAAE,cAAc;YACxB,SAAS,EAAE,iBAAiB;SAC7B,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,MAAM,IAAA,+CAAsB,kBACjD,KAAK,EACL,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC,EAChC,MAAM,IACH,IAAI,EACP,CAAC;QAEH,mEAAmE;QACnE,MAAM,KAAK,GAAG,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAE5C,uBACE,KAAK,IACF,cAAc,EACjB;IACJ,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"createDocumentRelevancyEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"createDocumentRelevancyEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,iCAAiC,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAI9E,MAAM,WAAW,8BACf,SAAQ,IAAI,CACV,iCAAiC,EACjC,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACD,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,uBAAuB,CAAC,CAAC;IACnF,IAAI,CAAC,EAAE,iCAAiC,CAAC,MAAM,CAAC,CAAC;IACjD,OAAO,CAAC,EAAE,iCAAiC,CAAC,SAAS,CAAC,CAAC;IACvD,cAAc,CAAC,EAAE,iCAAiC,CAAC,gBAAgB,CAAC,CAAC;CACtE;AAED;;GAEG;AACH,MAAM,WAAW,iCAAiC;IAChD,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,gCAAgC,CAC9C,UAAU,SAAS,MAAM,CACvB,MAAM,EACN,OAAO,CACR,GAAG,iCAAiC,EACrC,IAAI,EAAE,8BAA8B,GAAG,SAAS,CAAC,UAAU,CAAC,CAiB7D"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"createDocumentRelevancyEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"createDocumentRelevancyEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createDocumentRelevancyEvaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAsDA,4EAsBC;AA5ED,kGAG0D;AAG1D,mFAAgF;AAsBhF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,SAAgB,gCAAgC,CAK9C,IAAoC;IACpC,MAAM,EACJ,OAAO,GAAG,wDAA0B,EACpC,cAAc,GAAG,yDAA2B,EAC5C,qBAAqB,GAAG,UAAU,EAClC,IAAI,GAAG,oBAAoB,KAEzB,IAAI,EADH,IAAI,UACL,IAAI,EANF,8DAML,CAAO,CAAC;IAET,OAAO,IAAA,6DAA6B,gDAC/B,IAAI,KACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI,KACD,IAAI,EACP,CAAC;AACL,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"createHallucinationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"createHallucinationEvaluator.d.ts","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,iCAAiC,EAAE,MAAM,gBAAgB,CAAC;AAEnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAGpE,MAAM,WAAW,0BACf,SAAQ,IAAI,CACV,iCAAiC,EACjC,gBAAgB,GAAG,SAAS,GAAG,uBAAuB,GAAG,MAAM,CAChE;IACD,qBAAqB,CAAC,EAAE,iCAAiC,CAAC,uBAAuB,CAAC,CAAC;IACnF,IAAI,CAAC,EAAE,iCAAiC,CAAC,MAAM,CAAC,CAAC;IACjD,OAAO,CAAC,EAAE,iCAAiC,CAAC,SAAS,CAAC,CAAC;IACvD,cAAc,CAAC,EAAE,iCAAiC,CAAC,gBAAgB,CAAC,CAAC;CACtE;AAED;;GAEG;AACH,MAAM,MAAM,6BAA6B,GAAG;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AACF;;;;;GAKG;AACH,wBAAgB,4BAA4B,CAC1C,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,6BAA6B,EAC1E,IAAI,EAAE,0BAA0B,GAAG,uBAAuB,CAAC,UAAU,CAAC,CAgBvE"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"createHallucinationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"createHallucinationEvaluator.js","sourceRoot":"","sources":["../../../src/llm/createHallucinationEvaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAmCA,oEAkBC;AArDD,wFAGqD;AAIrD,mFAAgF;AAsBhF;;;;;GAKG;AACH,SAAgB,4BAA4B,CAE1C,IAAgC;IAChC,MAAM,EACJ,OAAO,GAAG,8CAAqB,EAC/B,cAAc,GAAG,+CAAsB,EACvC,qBAAqB,GAAG,UAAU,EAClC,IAAI,GAAG,eAAe,KAEpB,IAAI,EADH,IAAI,UACL,IAAI,EANF,8DAML,CAAO,CAAC;IACT,OAAO,IAAA,6DAA6B,gDAC/B,IAAI,KACP,cAAc;QACd,OAAO;QACP,qBAAqB;QACrB,IAAI,KACD,IAAI,EACP,CAAC;AACL,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"generateClassification.d.ts","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"generateClassification.d.ts","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAInD,MAAM,MAAM,YAAY,GAAG,OAAO,GAChC,aAAa,GACb,UAAU,GAAG;IACX;;OAEG;IACH,MAAM,EAAE,CAAC,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;IAC9B;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,CAAC;AACJ;;GAEG;AACH,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,YAAY,GACjB,OAAO,CAAC,oBAAoB,CAAC,CAyB/B"}
|
|
@@ -12,9 +12,9 @@ var __rest = (this && this.__rest) || function (s, e) {
|
|
|
12
12
|
};
|
|
13
13
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
14
|
exports.generateClassification = generateClassification;
|
|
15
|
+
const telemetry_1 = require("../telemetry");
|
|
15
16
|
const ai_1 = require("ai");
|
|
16
17
|
const zod_1 = require("zod");
|
|
17
|
-
const telemetry_1 = require("../telemetry");
|
|
18
18
|
/**
|
|
19
19
|
* A function that leverages an llm to perform a classification
|
|
20
20
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"generateClassification.js","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":";;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"generateClassification.js","sourceRoot":"","sources":["../../../src/llm/generateClassification.ts"],"names":[],"mappings":";;;;;;;;;;;;;AA0BA,wDA2BC;AArDD,4CAAsC;AAKtC,2BAAoC;AACpC,6BAAwB;AAiBxB;;GAEG;AACI,KAAK,UAAU,sBAAsB,CAC1C,IAAkB;;IAElB,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,iBAAiB,EAAE,SAAS,KAC7D,IAAI,EAD8D,MAAM,UACxE,IAAI,EADA,mEAAsE,CACtE,CAAC;IAEP,MAAM,sBAAsB,GAAG;QAC7B,SAAS,EAAE,MAAA,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,SAAS,mCAAI,IAAI;QACvC,UAAU,EAAE,wBAAwB;QACpC,MAAM,EAAE,MAAA,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,MAAM,mCAAI,kBAAM;KACpC,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,IAAA,mBAAc,kBACjC,KAAK;QACL,UAAU;QACV,iBAAiB,EACjB,MAAM,EAAE,OAAC,CAAC,MAAM,CAAC;YACf,WAAW,EAAE,OAAC,CAAC,MAAM,EAAE,EAAE,4EAA4E;YACrG,KAAK,EAAE,OAAC,CAAC,IAAI,CAAC,MAAM,CAAC;SACtB,CAAC,EACF,sBAAsB,IACnB,MAAM,EACT,CAAC;IACH,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;QAC1B,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW;KACvC,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"applyTemplate.d.ts","sourceRoot":"","sources":["../../../src/template/applyTemplate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"applyTemplate.d.ts","sourceRoot":"","sources":["../../../src/template/applyTemplate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAI/C;;;GAGG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE;IACnC,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,UAGA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"applyTemplate.js","sourceRoot":"","sources":["../../../src/template/applyTemplate.ts"],"names":[],"mappings":";;;;;
|
|
1
|
+
{"version":3,"file":"applyTemplate.js","sourceRoot":"","sources":["../../../src/template/applyTemplate.ts"],"names":[],"mappings":";;;;;AAQA,wCAMC;AAZD,wDAAgC;AAEhC;;;GAGG;AACH,SAAgB,cAAc,CAAC,IAG9B;IACC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;IACrC,OAAO,kBAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;AAC9C,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"getTemplateVariables.d.ts","sourceRoot":"","sources":["../../../src/template/getTemplateVariables.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"getTemplateVariables.d.ts","sourceRoot":"","sources":["../../../src/template/getTemplateVariables.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAI/C,KAAK,uBAAuB,GAAG;IAC7B,QAAQ,EAAE,QAAQ,CAAC;CACpB,CAAC;AACF;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,uBAAuB,GAAG,MAAM,EAAE,CAU5E"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"getTemplateVariables.js","sourceRoot":"","sources":["../../../src/template/getTemplateVariables.ts"],"names":[],"mappings":";;;;;
|
|
1
|
+
{"version":3,"file":"getTemplateVariables.js","sourceRoot":"","sources":["../../../src/template/getTemplateVariables.ts"],"names":[],"mappings":";;;;;AAYA,oDAUC;AApBD,wDAAgC;AAKhC;;;;GAIG;AACH,SAAgB,oBAAoB,CAAC,IAA6B;IAChE,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IAC1B,MAAM,aAAa,GAAG,kBAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC/C,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,YAAY,EAAE,EAAE;QAChD,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,GAAG,YAAY,CAAC;QACvC,IAAI,QAAQ,KAAK,MAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YACrD,GAAG,GAAG,CAAC,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;QACxB,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,EAAE,EAAc,CAAC,CAAC;AACrB,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { LanguageModel } from "ai";
|
|
2
1
|
import { WithTelemetry } from "./otel";
|
|
2
|
+
import { LanguageModel } from "ai";
|
|
3
3
|
/**
|
|
4
4
|
* A specific AI example that is under evaluation
|
|
5
5
|
*/
|
|
@@ -83,9 +83,9 @@ export interface CreateClassificationEvaluatorArgs extends CreateClassifierArgs,
|
|
|
83
83
|
}
|
|
84
84
|
export type EvaluatorFn<ExampleType extends Record<string, unknown>> = (args: ExampleType) => Promise<EvaluationResult>;
|
|
85
85
|
/**
|
|
86
|
-
* The
|
|
86
|
+
* The kind of the evaluation
|
|
87
87
|
*/
|
|
88
|
-
export type
|
|
88
|
+
export type EvaluationKind = "LLM" | "CODE";
|
|
89
89
|
/**
|
|
90
90
|
* The direction to optimize the numeric evaluation score
|
|
91
91
|
* E.x. "MAXIMIZE" means that the higher the score, the better the evaluation
|
|
@@ -100,9 +100,9 @@ interface EvaluatorDescription {
|
|
|
100
100
|
*/
|
|
101
101
|
name: string;
|
|
102
102
|
/**
|
|
103
|
-
* The
|
|
103
|
+
* The kind of the evaluation. Also known as the "kind" of evaluator.
|
|
104
104
|
*/
|
|
105
|
-
|
|
105
|
+
kind: EvaluationKind;
|
|
106
106
|
/**
|
|
107
107
|
* The direction to optimize the numeric evaluation score
|
|
108
108
|
* E.x. "MAXIMIZE" means that the higher the score, the better the evaluation
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../../../src/types/evals.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../../../src/types/evals.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAEvC,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAEnC;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,UAAU,EAAE,SAAS;IAClD,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,aAAa,CAAC;CACtB;AAGD,MAAM,WAAW,iBAAkB,SAAQ,OAAO;CAAG;AAErD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,aAAa;IAIzD,KAAK,EAAE,aAAa,CAAC;IACrB;;;OAGG;IACH,OAAO,EAAE,wBAAwB,CAAC;IAClC;;OAEG;IACH,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,mBAAmB;IAClC;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;CAC/C;AAED,MAAM,WAAW,iCACf,SAAQ,oBAAoB,EAC1B,mBAAmB;CAAG;AAE1B,MAAM,MAAM,WAAW,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CACrE,IAAI,EAAE,WAAW,KACd,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE/B;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,KAAK,GAAG,MAAM,CAAC;AAE5C;;;GAGG;AACH,MAAM,MAAM,qBAAqB,GAAG,UAAU,GAAG,UAAU,CAAC;AAE5D;;GAEG;AACH,UAAU,oBAAoB;IAC5B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;CAC/C;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS,CAAC,WAAW,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CACpE,SAAQ,oBAAoB;IAC5B;;OAEG;IACH,QAAQ,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;CACpC"}
|