@arizeai/phoenix-client 1.0.2 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/README.md +17 -5
  2. package/dist/esm/__generated__/api/v1.d.ts +1809 -295
  3. package/dist/esm/__generated__/api/v1.d.ts.map +1 -1
  4. package/dist/esm/experiments/runExperiment.d.ts +48 -13
  5. package/dist/esm/experiments/runExperiment.d.ts.map +1 -1
  6. package/dist/esm/experiments/runExperiment.js +139 -36
  7. package/dist/esm/experiments/runExperiment.js.map +1 -1
  8. package/dist/esm/schemas/llm/constants.d.ts +1 -1
  9. package/dist/esm/schemas/llm/converters.d.ts +4 -4
  10. package/dist/esm/schemas/llm/openai/converters.d.ts +1 -1
  11. package/dist/esm/schemas/llm/phoenixPrompt/converters.d.ts +8 -8
  12. package/dist/esm/schemas/llm/phoenixPrompt/messagePartSchemas.d.ts +5 -5
  13. package/dist/esm/schemas/llm/phoenixPrompt/messageSchemas.d.ts +8 -8
  14. package/dist/esm/schemas/llm/schemas.d.ts +4 -4
  15. package/dist/esm/spans/addSpanAnnotation.d.ts +39 -0
  16. package/dist/esm/spans/addSpanAnnotation.d.ts.map +1 -0
  17. package/dist/esm/spans/addSpanAnnotation.js +44 -0
  18. package/dist/esm/spans/addSpanAnnotation.js.map +1 -0
  19. package/dist/esm/spans/index.d.ts +3 -0
  20. package/dist/esm/spans/index.d.ts.map +1 -0
  21. package/dist/esm/spans/index.js +3 -0
  22. package/dist/esm/spans/index.js.map +1 -0
  23. package/dist/esm/spans/logSpanAnnotations.d.ts +51 -0
  24. package/dist/esm/spans/logSpanAnnotations.d.ts.map +1 -0
  25. package/dist/esm/spans/logSpanAnnotations.js +53 -0
  26. package/dist/esm/spans/logSpanAnnotations.js.map +1 -0
  27. package/dist/esm/spans/types.d.ts +43 -0
  28. package/dist/esm/spans/types.d.ts.map +1 -0
  29. package/dist/esm/spans/types.js +18 -0
  30. package/dist/esm/spans/types.js.map +1 -0
  31. package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
  32. package/dist/esm/types/annotations.d.ts +2 -1
  33. package/dist/esm/types/annotations.d.ts.map +1 -1
  34. package/dist/esm/types/experiments.d.ts +1 -7
  35. package/dist/esm/types/experiments.d.ts.map +1 -1
  36. package/dist/esm/utils/formatPromptMessages.d.ts +3 -1
  37. package/dist/esm/utils/formatPromptMessages.d.ts.map +1 -1
  38. package/dist/esm/utils/getPromptBySelector.d.ts.map +1 -1
  39. package/dist/src/__generated__/api/v1.d.ts +1809 -295
  40. package/dist/src/__generated__/api/v1.d.ts.map +1 -1
  41. package/dist/src/experiments/runExperiment.d.ts +48 -13
  42. package/dist/src/experiments/runExperiment.d.ts.map +1 -1
  43. package/dist/src/experiments/runExperiment.js +138 -36
  44. package/dist/src/experiments/runExperiment.js.map +1 -1
  45. package/dist/src/schemas/llm/constants.d.ts +1 -1
  46. package/dist/src/schemas/llm/converters.d.ts +4 -4
  47. package/dist/src/schemas/llm/openai/converters.d.ts +1 -1
  48. package/dist/src/schemas/llm/phoenixPrompt/converters.d.ts +8 -8
  49. package/dist/src/schemas/llm/phoenixPrompt/messagePartSchemas.d.ts +5 -5
  50. package/dist/src/schemas/llm/phoenixPrompt/messageSchemas.d.ts +8 -8
  51. package/dist/src/schemas/llm/schemas.d.ts +4 -4
  52. package/dist/src/spans/addSpanAnnotation.d.ts +39 -0
  53. package/dist/src/spans/addSpanAnnotation.d.ts.map +1 -0
  54. package/dist/src/spans/addSpanAnnotation.js +59 -0
  55. package/dist/src/spans/addSpanAnnotation.js.map +1 -0
  56. package/dist/src/spans/index.d.ts +3 -0
  57. package/dist/src/spans/index.d.ts.map +1 -0
  58. package/dist/src/spans/index.js +19 -0
  59. package/dist/src/spans/index.js.map +1 -0
  60. package/dist/src/spans/logSpanAnnotations.d.ts +51 -0
  61. package/dist/src/spans/logSpanAnnotations.d.ts.map +1 -0
  62. package/dist/src/spans/logSpanAnnotations.js +68 -0
  63. package/dist/src/spans/logSpanAnnotations.js.map +1 -0
  64. package/dist/src/spans/types.d.ts +43 -0
  65. package/dist/src/spans/types.d.ts.map +1 -0
  66. package/dist/src/spans/types.js +22 -0
  67. package/dist/src/spans/types.js.map +1 -0
  68. package/dist/src/types/annotations.d.ts +2 -1
  69. package/dist/src/types/annotations.d.ts.map +1 -1
  70. package/dist/src/types/experiments.d.ts +1 -7
  71. package/dist/src/types/experiments.d.ts.map +1 -1
  72. package/dist/src/utils/formatPromptMessages.d.ts +3 -1
  73. package/dist/src/utils/formatPromptMessages.d.ts.map +1 -1
  74. package/dist/src/utils/getPromptBySelector.d.ts.map +1 -1
  75. package/dist/tsconfig.tsbuildinfo +1 -1
  76. package/package.json +8 -2
  77. package/src/__generated__/api/v1.ts +1809 -295
  78. package/src/experiments/runExperiment.ts +211 -74
  79. package/src/spans/addSpanAnnotation.ts +59 -0
  80. package/src/spans/index.ts +2 -0
  81. package/src/spans/logSpanAnnotations.ts +71 -0
  82. package/src/spans/types.ts +60 -0
  83. package/src/types/annotations.ts +4 -1
  84. package/src/types/experiments.ts +1 -7
@@ -0,0 +1,59 @@
1
+ "use strict";
2
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
+ return new (P || (P = Promise))(function (resolve, reject) {
5
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
9
+ });
10
+ };
11
+ Object.defineProperty(exports, "__esModule", { value: true });
12
+ exports.addSpanAnnotation = addSpanAnnotation;
13
+ const client_1 = require("../client");
14
+ const types_1 = require("./types");
15
+ /**
16
+ * Add an annotation to a span.
17
+ *
18
+ * The annotation can be of type "LLM", "CODE", or "HUMAN" and can include a label, score, and metadata.
19
+ * If an identifier is provided and an annotation with that identifier already exists, it will be updated.
20
+ *
21
+ * @param params - The parameters to add a span annotation
22
+ * @returns The ID of the created or updated annotation
23
+ *
24
+ * @example
25
+ * ```ts
26
+ * const result = await addSpanAnnotation({
27
+ * spanAnnotation: {
28
+ * spanId: "123abc",
29
+ * name: "quality_score",
30
+ * label: "good",
31
+ * score: 0.95,
32
+ * annotatorKind: "LLM",
33
+ * identifier: "custom_id_123",
34
+ * metadata: {
35
+ * model: "gpt-4"
36
+ * }
37
+ * }
38
+ * });
39
+ * ```
40
+ */
41
+ function addSpanAnnotation(_a) {
42
+ return __awaiter(this, arguments, void 0, function* ({ client: _client, spanAnnotation, }) {
43
+ var _b, _c;
44
+ const client = _client !== null && _client !== void 0 ? _client : (0, client_1.createClient)();
45
+ const { data, error } = yield client.POST("/v1/span_annotations", {
46
+ body: {
47
+ data: [(0, types_1.toSpanAnnotationData)(spanAnnotation)],
48
+ },
49
+ });
50
+ if (error) {
51
+ throw new Error(`Failed to add span annotation: ${error}`);
52
+ }
53
+ if (!((_c = (_b = data === null || data === void 0 ? void 0 : data.data) === null || _b === void 0 ? void 0 : _b[0]) === null || _c === void 0 ? void 0 : _c.id)) {
54
+ throw new Error("No annotation ID returned from server");
55
+ }
56
+ return data.data[0];
57
+ });
58
+ }
59
+ //# sourceMappingURL=addSpanAnnotation.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"addSpanAnnotation.js","sourceRoot":"","sources":["../../../src/spans/addSpanAnnotation.ts"],"names":[],"mappings":";;;;;;;;;;;AAqCA,8CAqBC;AA1DD,sCAAyC;AAEzC,mCAA+D;AAS/D;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,SAAsB,iBAAiB;yDAAC,EACtC,MAAM,EAAE,OAAO,EACf,cAAc,GACU;;QACxB,MAAM,MAAM,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,IAAA,qBAAY,GAAE,CAAC;QAEzC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE;YAChE,IAAI,EAAE;gBACJ,IAAI,EAAE,CAAC,IAAA,4BAAoB,EAAC,cAAc,CAAC,CAAC;aAC7C;SACF,CAAC,CAAC;QAEH,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,kCAAkC,KAAK,EAAE,CAAC,CAAC;QAC7D,CAAC;QAED,IAAI,CAAC,CAAA,MAAA,MAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,IAAI,0CAAG,CAAC,CAAC,0CAAE,EAAE,CAAA,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;QAED,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtB,CAAC;CAAA"}
@@ -0,0 +1,3 @@
1
+ export * from "./addSpanAnnotation";
2
+ export * from "./logSpanAnnotations";
3
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/spans/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC;AACpC,cAAc,sBAAsB,CAAC"}
@@ -0,0 +1,19 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
+ };
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ __exportStar(require("./addSpanAnnotation"), exports);
18
+ __exportStar(require("./logSpanAnnotations"), exports);
19
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/spans/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,sDAAoC;AACpC,uDAAqC"}
@@ -0,0 +1,51 @@
1
+ import { ClientFn } from "../types/core";
2
+ import { SpanAnnotation } from "./types";
3
+ /**
4
+ * Parameters to log multiple span annotations
5
+ */
6
+ interface LogSpanAnnotationsParams extends ClientFn {
7
+ /**
8
+ * The span annotations to log
9
+ */
10
+ spanAnnotations: SpanAnnotation[];
11
+ }
12
+ /**
13
+ * Log multiple span annotations in a single request.
14
+ *
15
+ * Each annotation can be of type "LLM", "CODE", or "HUMAN" and can include a label, score, and metadata.
16
+ * If an identifier is provided and an annotation with that identifier already exists, it will be updated.
17
+ *
18
+ * @param params - The parameters to log span annotations
19
+ * @returns The IDs of the created or updated annotations
20
+ *
21
+ * @example
22
+ * ```ts
23
+ * const results = await logSpanAnnotations({
24
+ * spanAnnotations: [
25
+ * {
26
+ * spanId: "123abc",
27
+ * name: "quality_score",
28
+ * label: "good",
29
+ * score: 0.95,
30
+ * annotatorKind: "LLM",
31
+ * identifier: "custom_id_123",
32
+ * metadata: {
33
+ * model: "gpt-4"
34
+ * }
35
+ * },
36
+ * {
37
+ * spanId: "456def",
38
+ * name: "sentiment",
39
+ * label: "positive",
40
+ * score: 0.8,
41
+ * annotatorKind: "CODE"
42
+ * }
43
+ * ]
44
+ * });
45
+ * ```
46
+ */
47
+ export declare function logSpanAnnotations({ client: _client, spanAnnotations, }: LogSpanAnnotationsParams): Promise<{
48
+ id: string;
49
+ }[]>;
50
+ export {};
51
+ //# sourceMappingURL=logSpanAnnotations.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"logSpanAnnotations.d.ts","sourceRoot":"","sources":["../../../src/spans/logSpanAnnotations.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,cAAc,EAAwB,MAAM,SAAS,CAAC;AAE/D;;GAEG;AACH,UAAU,wBAAyB,SAAQ,QAAQ;IACjD;;OAEG;IACH,eAAe,EAAE,cAAc,EAAE,CAAC;CACnC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,wBAAsB,kBAAkB,CAAC,EACvC,MAAM,EAAE,OAAO,EACf,eAAe,GAChB,EAAE,wBAAwB,GAAG,OAAO,CAAC;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,EAAE,CAAC,CAkBtD"}
@@ -0,0 +1,68 @@
1
+ "use strict";
2
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
+ return new (P || (P = Promise))(function (resolve, reject) {
5
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
9
+ });
10
+ };
11
+ Object.defineProperty(exports, "__esModule", { value: true });
12
+ exports.logSpanAnnotations = logSpanAnnotations;
13
+ const client_1 = require("../client");
14
+ const types_1 = require("./types");
15
+ /**
16
+ * Log multiple span annotations in a single request.
17
+ *
18
+ * Each annotation can be of type "LLM", "CODE", or "HUMAN" and can include a label, score, and metadata.
19
+ * If an identifier is provided and an annotation with that identifier already exists, it will be updated.
20
+ *
21
+ * @param params - The parameters to log span annotations
22
+ * @returns The IDs of the created or updated annotations
23
+ *
24
+ * @example
25
+ * ```ts
26
+ * const results = await logSpanAnnotations({
27
+ * spanAnnotations: [
28
+ * {
29
+ * spanId: "123abc",
30
+ * name: "quality_score",
31
+ * label: "good",
32
+ * score: 0.95,
33
+ * annotatorKind: "LLM",
34
+ * identifier: "custom_id_123",
35
+ * metadata: {
36
+ * model: "gpt-4"
37
+ * }
38
+ * },
39
+ * {
40
+ * spanId: "456def",
41
+ * name: "sentiment",
42
+ * label: "positive",
43
+ * score: 0.8,
44
+ * annotatorKind: "CODE"
45
+ * }
46
+ * ]
47
+ * });
48
+ * ```
49
+ */
50
+ function logSpanAnnotations(_a) {
51
+ return __awaiter(this, arguments, void 0, function* ({ client: _client, spanAnnotations, }) {
52
+ var _b;
53
+ const client = _client !== null && _client !== void 0 ? _client : (0, client_1.createClient)();
54
+ const { data, error } = yield client.POST("/v1/span_annotations", {
55
+ body: {
56
+ data: spanAnnotations.map(types_1.toSpanAnnotationData),
57
+ },
58
+ });
59
+ if (error) {
60
+ throw new Error(`Failed to log span annotations: ${error}`);
61
+ }
62
+ if (!((_b = data === null || data === void 0 ? void 0 : data.data) === null || _b === void 0 ? void 0 : _b.length)) {
63
+ throw new Error("No annotation IDs returned from server");
64
+ }
65
+ return data.data;
66
+ });
67
+ }
68
+ //# sourceMappingURL=logSpanAnnotations.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"logSpanAnnotations.js","sourceRoot":"","sources":["../../../src/spans/logSpanAnnotations.ts"],"names":[],"mappings":";;;;;;;;;;;AAiDA,gDAqBC;AAtED,sCAAyC;AAEzC,mCAA+D;AAY/D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,SAAsB,kBAAkB;yDAAC,EACvC,MAAM,EAAE,OAAO,EACf,eAAe,GACU;;QACzB,MAAM,MAAM,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,IAAA,qBAAY,GAAE,CAAC;QAEzC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE;YAChE,IAAI,EAAE;gBACJ,IAAI,EAAE,eAAe,CAAC,GAAG,CAAC,4BAAoB,CAAC;aAChD;SACF,CAAC,CAAC;QAEH,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,EAAE,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,CAAC,CAAA,MAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,IAAI,0CAAE,MAAM,CAAA,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC5D,CAAC;QAED,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;CAAA"}
@@ -0,0 +1,43 @@
1
+ import { paths } from "../__generated__/api/v1";
2
+ type SpanAnnotationData = paths["/v1/span_annotations"]["post"]["requestBody"]["content"]["application/json"]["data"][0];
3
+ /**
4
+ * Parameters for a single span annotation
5
+ */
6
+ export interface SpanAnnotation {
7
+ /**
8
+ * The OpenTelemetry Span ID (hex format without 0x prefix)
9
+ */
10
+ spanId: string;
11
+ /**
12
+ * The name of the annotation
13
+ */
14
+ name: string;
15
+ /**
16
+ * The label assigned by the annotation
17
+ */
18
+ label?: string;
19
+ /**
20
+ * The score assigned by the annotation
21
+ */
22
+ score?: number;
23
+ /**
24
+ * The identifier of the annotation. If provided, the annotation will be updated if it already exists.
25
+ */
26
+ identifier?: string;
27
+ /**
28
+ * Metadata for the annotation
29
+ */
30
+ metadata?: Record<string, unknown>;
31
+ /**
32
+ * The kind of annotator used for the annotation
33
+ * Can be "HUMAN", "LLM", or "CODE"
34
+ * @default "HUMAN"
35
+ */
36
+ annotatorKind?: SpanAnnotationData["annotator_kind"];
37
+ }
38
+ /**
39
+ * Convert a SpanAnnotation to the API format
40
+ */
41
+ export declare function toSpanAnnotationData(annotation: SpanAnnotation): SpanAnnotationData;
42
+ export {};
43
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/spans/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEhD,KAAK,kBAAkB,GACrB,KAAK,CAAC,sBAAsB,CAAC,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,CAAC,SAAS,CAAC,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAEjG;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC;;;;OAIG;IACH,aAAa,CAAC,EAAE,kBAAkB,CAAC,gBAAgB,CAAC,CAAC;CACtD;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAClC,UAAU,EAAE,cAAc,GACzB,kBAAkB,CAapB"}
@@ -0,0 +1,22 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.toSpanAnnotationData = toSpanAnnotationData;
4
+ /**
5
+ * Convert a SpanAnnotation to the API format
6
+ */
7
+ function toSpanAnnotationData(annotation) {
8
+ var _a, _b, _c, _d, _e;
9
+ return {
10
+ span_id: annotation.spanId,
11
+ name: annotation.name,
12
+ annotator_kind: (_a = annotation.annotatorKind) !== null && _a !== void 0 ? _a : "HUMAN",
13
+ result: {
14
+ label: (_b = annotation.label) !== null && _b !== void 0 ? _b : null,
15
+ score: (_c = annotation.score) !== null && _c !== void 0 ? _c : null,
16
+ explanation: null,
17
+ },
18
+ metadata: (_d = annotation.metadata) !== null && _d !== void 0 ? _d : null,
19
+ identifier: (_e = annotation.identifier) !== null && _e !== void 0 ? _e : "",
20
+ };
21
+ }
22
+ //# sourceMappingURL=types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/spans/types.ts"],"names":[],"mappings":";;AA4CA,oDAeC;AAlBD;;GAEG;AACH,SAAgB,oBAAoB,CAClC,UAA0B;;IAE1B,OAAO;QACL,OAAO,EAAE,UAAU,CAAC,MAAM;QAC1B,IAAI,EAAE,UAAU,CAAC,IAAI;QACrB,cAAc,EAAE,MAAA,UAAU,CAAC,aAAa,mCAAI,OAAO;QACnD,MAAM,EAAE;YACN,KAAK,EAAE,MAAA,UAAU,CAAC,KAAK,mCAAI,IAAI;YAC/B,KAAK,EAAE,MAAA,UAAU,CAAC,KAAK,mCAAI,IAAI;YAC/B,WAAW,EAAE,IAAI;SAClB;QACD,QAAQ,EAAE,MAAA,UAAU,CAAC,QAAQ,mCAAI,IAAI;QACrC,UAAU,EAAE,MAAA,UAAU,CAAC,UAAU,mCAAI,EAAE;KACxC,CAAC;AACJ,CAAC"}
@@ -1,2 +1,3 @@
1
- export type AnnotatorKind = "HUMAN" | "LLM";
1
+ import { components } from "../__generated__/api/v1";
2
+ export type AnnotatorKind = components["schemas"]["SpanAnnotationData"]["annotator_kind"];
2
3
  //# sourceMappingURL=annotations.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"annotations.d.ts","sourceRoot":"","sources":["../../../src/types/annotations.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,aAAa,GAAG,OAAO,GAAG,KAAK,CAAC"}
1
+ {"version":3,"file":"annotations.d.ts","sourceRoot":"","sources":["../../../src/types/annotations.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAErD,MAAM,MAAM,aAAa,GACvB,UAAU,CAAC,SAAS,CAAC,CAAC,oBAAoB,CAAC,CAAC,gBAAgB,CAAC,CAAC"}
@@ -7,7 +7,6 @@ import { Example } from "./datasets";
7
7
  export interface Experiment extends Node {
8
8
  datasetId: string;
9
9
  datasetVersionId: string;
10
- repetitions: number;
11
10
  /**
12
11
  * The project under which the experiment task traces are recorded
13
12
  */
@@ -29,7 +28,6 @@ export interface ExperimentRun extends Node {
29
28
  */
30
29
  experimentId: string;
31
30
  datasetExampleId: string;
32
- repetitionNumber: number;
33
31
  output?: string | Record<string, unknown> | null;
34
32
  error: string | null;
35
33
  traceId: string | null;
@@ -54,6 +52,7 @@ export type EvaluatorParams = {
54
52
  };
55
53
  export type Evaluator = {
56
54
  name: string;
55
+ kind: AnnotatorKind;
57
56
  evaluate: (args: EvaluatorParams) => Promise<EvaluationResult> | EvaluationResult;
58
57
  };
59
58
  export type EvaluationResult = {
@@ -86,10 +85,5 @@ export interface ExperimentParameters {
86
85
  * The number of examples to run the experiment on
87
86
  */
88
87
  nExamples: number;
89
- /**
90
- * The number of repetitions to run the experiment
91
- * e.g. the number of times to run the task
92
- */
93
- nRepetitions: number;
94
88
  }
95
89
  //# sourceMappingURL=experiments.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"experiments.d.ts","sourceRoot":"","sources":["../../../src/types/experiments.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AAC9B,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC;;GAEG;AACH,MAAM,WAAW,UAAW,SAAQ,IAAI;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,aAAc,SAAQ,UAAU;IAC/C,MAAM,EAAE,oBAAoB,CAAC;IAC7B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IACpC,cAAc,CAAC,EAAE,uBAAuB,EAAE,CAAC;CAC5C;AAED;;GAEG;AACH,MAAM,WAAW,aAAc,SAAQ,IAAI;IACzC,SAAS,EAAE,IAAI,CAAC;IAChB,OAAO,EAAE,IAAI,CAAC;IACd;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACjD,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB;AAED,MAAM,MAAM,eAAe,GAAG;IAC5B;;OAEG;IACH,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IACxB;;OAEG;IACH,MAAM,EAAE,UAAU,CAAC;IACnB;;OAEG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC7B;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,CACR,IAAI,EAAE,eAAe,KAClB,OAAO,CAAC,gBAAgB,CAAC,GAAG,gBAAgB,CAAC;CACnD,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B,CAAC;AAEF,MAAM,WAAW,uBAAwB,SAAQ,IAAI;IACnD,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,IAAI,CAAC;IAChB,OAAO,EAAE,IAAI,CAAC;IACd;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,aAAa,CAAC;IAC7B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAAC;IAChC;;;OAGG;IACH,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;AAEnE,MAAM,MAAM,cAAc,GAAG,CAC3B,OAAO,EAAE,OAAO,KACb,OAAO,CAAC,UAAU,CAAC,GAAG,UAAU,CAAC;AAEtC,MAAM,WAAW,oBAAoB;IACnC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,YAAY,EAAE,MAAM,CAAC;CACtB"}
1
+ {"version":3,"file":"experiments.d.ts","sourceRoot":"","sources":["../../../src/types/experiments.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AAC9B,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC;;GAEG;AACH,MAAM,WAAW,UAAW,SAAQ,IAAI;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,aAAc,SAAQ,UAAU;IAC/C,MAAM,EAAE,oBAAoB,CAAC;IAC7B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IACpC,cAAc,CAAC,EAAE,uBAAuB,EAAE,CAAC;CAC5C;AAED;;GAEG;AACH,MAAM,WAAW,aAAc,SAAQ,IAAI;IACzC,SAAS,EAAE,IAAI,CAAC;IAChB,OAAO,EAAE,IAAI,CAAC;IACd;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACjD,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB;AAED,MAAM,MAAM,eAAe,GAAG;IAC5B;;OAEG;IACH,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IACxB;;OAEG;IACH,MAAM,EAAE,UAAU,CAAC;IACnB;;OAEG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC7B;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,aAAa,CAAC;IACpB,QAAQ,EAAE,CACR,IAAI,EAAE,eAAe,KAClB,OAAO,CAAC,gBAAgB,CAAC,GAAG,gBAAgB,CAAC;CACnD,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B,CAAC;AAEF,MAAM,WAAW,uBAAwB,SAAQ,IAAI;IACnD,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,IAAI,CAAC;IAChB,OAAO,EAAE,IAAI,CAAC;IACd;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,aAAa,CAAC;IAC7B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAAC;IAChC;;;OAGG;IACH,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;AAEnE,MAAM,MAAM,cAAc,GAAG,CAC3B,OAAO,EAAE,OAAO,KACb,OAAO,CAAC,UAAU,CAAC,GAAG,UAAU,CAAC;AAEtC,MAAM,WAAW,oBAAoB;IACnC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;CACnB"}
@@ -19,7 +19,9 @@ export declare function formatPromptMessages(format: PromptTemplateFormat, promp
19
19
  } | {
20
20
  type: "tool_result";
21
21
  tool_call_id: string;
22
- tool_result: boolean | number | string | Record<string, unknown> | unknown[] | null;
22
+ tool_result: boolean | number | string | {
23
+ [key: string]: unknown;
24
+ } | unknown[] | null;
23
25
  })[];
24
26
  role: "user" | "assistant" | "model" | "ai" | "tool" | "system" | "developer";
25
27
  }[];
@@ -1 +1 @@
1
- {"version":3,"file":"formatPromptMessages.d.ts","sourceRoot":"","sources":["../../../src/utils/formatPromptMessages.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAClD,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAO3E;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,oBAAoB,EAC5B,cAAc,EAAE,iBAAiB,EAAE,EACnC,SAAS,GAAE,SAAc;;;;;;;mBA4Dwi3B,uDAAsB;;;;;;;IAlBxl3B"}
1
+ {"version":3,"file":"formatPromptMessages.d.ts","sourceRoot":"","sources":["../../../src/utils/formatPromptMessages.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAClD,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAO3E;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,oBAAoB,EAC5B,cAAc,EAAE,iBAAiB,EAAE,EACnC,SAAS,GAAE,SAAc;;;;;;;mBA4DkxgD,uDAAsB;;;;;;;;;IAlBl0gD"}
@@ -1 +1 @@
1
- {"version":3,"file":"getPromptBySelector.d.ts","sourceRoot":"","sources":["../../../src/utils/getPromptBySelector.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAGzC;;GAEG;AACH,MAAM,MAAM,yBAAyB,GAAG,QAAQ,GAAG;IACjD;;OAEG;IACH,MAAM,EAAE,cAAc,CAAC;CACxB,CAAC;AAEF;;;;;;;;;;GAUG;AACH,wBAAsB,mBAAmB,CAAC,EACxC,MAAM,EAAE,OAAO,EACf,MAAM,GACP,EAAE,yBAAyB;;oBAkDk6uB,uDAAsB;;cAAkI,uDAAsB,yBAAwB,uDAAsB;mBAAoD,uDAAsB;qBAAoD,uDAAsB;2BAAqG,uDAAsB,uCAAsC,uDAAsB,4CAA2C,uDAAsB,0CAAyC,uDAAsB;YAAyD,uDAAsB;sBAAwF,uDAAsB;;UADjywB"}
1
+ {"version":3,"file":"getPromptBySelector.d.ts","sourceRoot":"","sources":["../../../src/utils/getPromptBySelector.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAGzC;;GAEG;AACH,MAAM,MAAM,yBAAyB,GAAG,QAAQ,GAAG;IACjD;;OAEG;IACH,MAAM,EAAE,cAAc,CAAC;CACxB,CAAC;AAEF;;;;;;;;;;GAUG;AACH,wBAAsB,mBAAmB,CAAC,EACxC,MAAM,EAAE,OAAO,EACf,MAAM,GACP,EAAE,yBAAyB;;oBAkD4kzC,uDAAsB;;cAAkI,uDAAsB,yBAAwB,uDAAsB;mBAAoD,uDAAsB;qBAAoD,uDAAsB;2BAAqG,uDAAsB,uCAAsC,uDAAsB,4CAA2C,uDAAsB,0CAAyC,uDAAsB;YAAyD,uDAAsB;sBAAwF,uDAAsB;;UAD380C"}