@botpress/evals 1.19.0-beta.14 → 1.19.0-beta.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/dist/client.d.ts +9 -9
  2. package/dist/client.d.ts.map +1 -1
  3. package/dist/client.js +74 -20
  4. package/dist/client.js.map +4 -4
  5. package/dist/definition.d.ts +2 -3
  6. package/dist/definition.d.ts.map +1 -1
  7. package/dist/definition.js.map +2 -2
  8. package/dist/graders/index.js +74 -21
  9. package/dist/graders/index.js.map +2 -2
  10. package/dist/graders/llm.d.ts +12 -12
  11. package/dist/graders/llm.d.ts.map +1 -1
  12. package/dist/graders/llm.js +75 -30
  13. package/dist/graders/llm.js.map +2 -2
  14. package/dist/graders/response.js +65 -16
  15. package/dist/graders/response.js.map +2 -2
  16. package/dist/index.d.ts +1 -3
  17. package/dist/index.d.ts.map +1 -1
  18. package/dist/index.js +0 -10
  19. package/dist/index.js.map +3 -3
  20. package/dist/loader.js +39 -1
  21. package/dist/loader.js.map +3 -3
  22. package/dist/runner.d.ts +1 -5
  23. package/dist/runner.d.ts.map +1 -1
  24. package/dist/runner.js +217 -141
  25. package/dist/runner.js.map +4 -4
  26. package/dist/{spans/sse-collector.d.ts → sse-collector.d.ts} +4 -6
  27. package/dist/sse-collector.d.ts.map +1 -0
  28. package/dist/{spans/sse-collector.js → sse-collector.js} +49 -17
  29. package/dist/sse-collector.js.map +7 -0
  30. package/dist/trace.d.ts.map +1 -0
  31. package/dist/transformer.d.ts +2 -2
  32. package/dist/transformer.d.ts.map +1 -1
  33. package/dist/transformer.js +1 -1
  34. package/dist/transformer.js.map +2 -2
  35. package/dist/types.d.ts +33 -38
  36. package/dist/types.d.ts.map +1 -1
  37. package/dist/types.js.map +2 -2
  38. package/package.json +6 -66
  39. package/dist/manifest.d.ts +0 -13
  40. package/dist/manifest.d.ts.map +0 -1
  41. package/dist/spans/index.d.ts +0 -5
  42. package/dist/spans/index.d.ts.map +0 -1
  43. package/dist/spans/index.js +0 -642
  44. package/dist/spans/index.js.map +0 -7
  45. package/dist/spans/local-span-source.d.ts +0 -16
  46. package/dist/spans/local-span-source.d.ts.map +0 -1
  47. package/dist/spans/local-span-source.js +0 -478
  48. package/dist/spans/local-span-source.js.map +0 -7
  49. package/dist/spans/span-source.d.ts +0 -20
  50. package/dist/spans/span-source.d.ts.map +0 -1
  51. package/dist/spans/span-source.js +0 -1
  52. package/dist/spans/span-source.js.map +0 -7
  53. package/dist/spans/sse-collector.d.ts.map +0 -1
  54. package/dist/spans/sse-collector.js.map +0 -7
  55. package/dist/spans/trace.d.ts.map +0 -1
  56. package/dist/spans/vortex-span-source.d.ts +0 -34
  57. package/dist/spans/vortex-span-source.d.ts.map +0 -1
  58. package/dist/spans/vortex-span-source.js +0 -166
  59. package/dist/spans/vortex-span-source.js.map +0 -7
  60. package/dist/stores/eval-definition-loader.d.ts +0 -4
  61. package/dist/stores/eval-definition-loader.d.ts.map +0 -1
  62. package/dist/stores/eval-definition-loader.js +0 -104
  63. package/dist/stores/eval-definition-loader.js.map +0 -7
  64. package/dist/stores/eval-store.d.ts +0 -61
  65. package/dist/stores/eval-store.d.ts.map +0 -1
  66. package/dist/stores/eval-store.js +0 -1
  67. package/dist/stores/eval-store.js.map +0 -7
  68. package/dist/stores/index.d.ts +0 -8
  69. package/dist/stores/index.d.ts.map +0 -1
  70. package/dist/stores/index.js +0 -1118
  71. package/dist/stores/index.js.map +0 -7
  72. package/dist/stores/local-eval-store.d.ts +0 -66
  73. package/dist/stores/local-eval-store.d.ts.map +0 -1
  74. package/dist/stores/local-eval-store.js +0 -557
  75. package/dist/stores/local-eval-store.js.map +0 -7
  76. package/dist/stores/vortex-eval-store-entry.d.ts +0 -6
  77. package/dist/stores/vortex-eval-store-entry.d.ts.map +0 -1
  78. package/dist/stores/vortex-eval-store-entry.js +0 -664
  79. package/dist/stores/vortex-eval-store-entry.js.map +0 -7
  80. package/dist/stores/vortex-eval-store.d.ts +0 -78
  81. package/dist/stores/vortex-eval-store.d.ts.map +0 -1
  82. package/dist/stores/vortex-eval-store.js +0 -562
  83. package/dist/stores/vortex-eval-store.js.map +0 -7
  84. /package/dist/{spans/trace.d.ts → trace.d.ts} +0 -0
  85. /package/dist/{spans/trace.js → trace.js} +0 -0
  86. /package/dist/{spans/trace.js.map → trace.js.map} +0 -0
@@ -1,61 +0,0 @@
1
- import type { EvalDefinition } from '../definition';
2
- import type { EvalFilter, EvalProgressEvent, EvalReport, EvalRunReport } from '../types';
3
- export interface EvalSummary {
4
- name: string;
5
- description?: string;
6
- tags: string[];
7
- type: 'capability' | 'regression';
8
- turnCount: number;
9
- hasOutcome: boolean;
10
- }
11
- export interface EvalRunSummary {
12
- id: string;
13
- timestamp: string;
14
- passed: number;
15
- failed: number;
16
- total: number;
17
- duration: number;
18
- botDuration: number;
19
- evalDuration: number;
20
- filter?: EvalFilter;
21
- evalNames: string[];
22
- aborted: boolean;
23
- }
24
- export interface EvalReportHistoryEntry {
25
- runId: string;
26
- timestamp: string;
27
- report: EvalReport;
28
- totalEvalsInRun: number;
29
- filter?: EvalFilter;
30
- }
31
- export interface EvalWatchOptions {
32
- runId?: string;
33
- workflowId?: string;
34
- }
35
- export interface EvalStore {
36
- listEvals(filter?: EvalFilter): Promise<EvalSummary[]>;
37
- getEval(name: string): Promise<EvalDefinition | null>;
38
- createRun(runType?: string, metadata?: Record<string, unknown>): Promise<string>;
39
- addRunResults(runId: string, evalReport: EvalReport): Promise<void>;
40
- completeRun(runId: string, report: EvalRunReport): Promise<void>;
41
- loadRunResult(runId: string): Promise<EvalRunReport | null>;
42
- getLatestRun(): Promise<EvalRunReport | null>;
43
- listRunSummaries(opts?: {
44
- limit?: number;
45
- since?: number;
46
- }): Promise<EvalRunSummary[]>;
47
- listEvalReportsByName(evalName: string, opts?: {
48
- limit?: number;
49
- since?: number;
50
- }): Promise<EvalReportHistoryEntry[]>;
51
- listEvalReportsBulk(opts?: {
52
- perEval?: number;
53
- since?: number;
54
- }): Promise<Record<string, EvalReportHistoryEntry[]>>;
55
- watchRun(signal?: AbortSignal, options?: EvalWatchOptions): AsyncIterable<EvalProgressEvent>;
56
- getRunnerState(): Promise<{
57
- running: boolean;
58
- runId: string | null;
59
- }>;
60
- }
61
- //# sourceMappingURL=eval-store.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"eval-store.d.ts","sourceRoot":"","sources":["../../src/stores/eval-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AACnD,OAAO,KAAK,EAAE,UAAU,EAAE,iBAAiB,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AAIxF,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,IAAI,EAAE,MAAM,EAAE,CAAA;IACd,IAAI,EAAE,YAAY,GAAG,YAAY,CAAA;IACjC,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,OAAO,CAAA;CACpB;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAA;IACV,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,CAAC,EAAE,UAAU,CAAA;IACnB,SAAS,EAAE,MAAM,EAAE,CAAA;IACnB,OAAO,EAAE,OAAO,CAAA;CACjB;AAED,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,UAAU,CAAA;IAClB,eAAe,EAAE,MAAM,CAAA;IACvB,MAAM,CAAC,EAAE,UAAU,CAAA;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAID,MAAM,WAAW,SAAS;IAExB,SAAS,CAAC,MAAM,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAAA;IACtD,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAAA;IAGrD,SAAS,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IAChF,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACnE,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAGhE,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAAA;IAC3D,YAAY,IAAI,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAAA;IAC7C,gBAAgB,CAAC,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAAA;IACtF,qBAAqB,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,sBAAsB,EAAE,CAAC,CAAA;IACrH,mBAAmB,CAAC,IAAI,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,sBAAsB,EAAE,CAAC,CAAC,CAAA;IAGnH,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,aAAa,CAAC,iBAAiB,CAAC,CAAA;IAG5F,cAAc,IAAI,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC,CAAA;CACtE"}
@@ -1 +0,0 @@
1
- //# sourceMappingURL=eval-store.js.map
@@ -1,7 +0,0 @@
1
- {
2
- "version": 3,
3
- "sources": [],
4
- "sourcesContent": [],
5
- "mappings": "",
6
- "names": []
7
- }
@@ -1,8 +0,0 @@
1
- export type { EvalStore, EvalSummary, EvalRunSummary, EvalReportHistoryEntry, EvalWatchOptions } from './eval-store';
2
- export { LocalEvalStore, getLocalEvalStore, closeLocalEvalStores } from './local-eval-store';
3
- export type { LocalEvalStoreConfig } from './local-eval-store';
4
- export { VortexEvalStore } from './vortex-eval-store';
5
- export type { VortexEvalStoreConfig } from './vortex-eval-store';
6
- export { createDiskEvalLoader } from './eval-definition-loader';
7
- export type { EvalDefinitionLoader } from './eval-definition-loader';
8
- //# sourceMappingURL=index.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/stores/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,SAAS,EAAE,WAAW,EAAE,cAAc,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AACpH,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AAC5F,YAAY,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AAC9D,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAA;AACrD,YAAY,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAA;AAChE,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAA;AAC/D,YAAY,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAA"}