@hasna/evals 0.1.24 → 0.1.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/dist/adapters/adapters.test.d.ts +2 -0
  2. package/dist/adapters/adapters.test.d.ts.map +1 -0
  3. package/dist/adapters/anthropic-openai.test.d.ts +2 -0
  4. package/dist/adapters/anthropic-openai.test.d.ts.map +1 -0
  5. package/dist/adapters/anthropic.d.ts +4 -0
  6. package/dist/adapters/anthropic.d.ts.map +1 -0
  7. package/dist/adapters/cli.d.ts +4 -0
  8. package/dist/adapters/cli.d.ts.map +1 -0
  9. package/dist/adapters/function.d.ts +4 -0
  10. package/dist/adapters/function.d.ts.map +1 -0
  11. package/dist/adapters/http-cli-coverage.test.d.ts +2 -0
  12. package/dist/adapters/http-cli-coverage.test.d.ts.map +1 -0
  13. package/dist/adapters/http.d.ts +16 -0
  14. package/dist/adapters/http.d.ts.map +1 -0
  15. package/dist/adapters/mcp-adapter.test.d.ts +2 -0
  16. package/dist/adapters/mcp-adapter.test.d.ts.map +1 -0
  17. package/dist/adapters/mcp.d.ts +4 -0
  18. package/dist/adapters/mcp.d.ts.map +1 -0
  19. package/dist/adapters/openai.d.ts +4 -0
  20. package/dist/adapters/openai.d.ts.map +1 -0
  21. package/dist/cli/adapter-parser.d.ts +3 -0
  22. package/dist/cli/adapter-parser.d.ts.map +1 -0
  23. package/dist/cli/cli.test.d.ts +2 -0
  24. package/dist/cli/cli.test.d.ts.map +1 -0
  25. package/dist/cli/commands/calibrate.d.ts +3 -0
  26. package/dist/cli/commands/calibrate.d.ts.map +1 -0
  27. package/dist/cli/commands/capture.d.ts +3 -0
  28. package/dist/cli/commands/capture.d.ts.map +1 -0
  29. package/dist/cli/commands/ci.d.ts +3 -0
  30. package/dist/cli/commands/ci.d.ts.map +1 -0
  31. package/dist/cli/commands/compare.d.ts +5 -0
  32. package/dist/cli/commands/compare.d.ts.map +1 -0
  33. package/dist/cli/commands/compare.test.d.ts +2 -0
  34. package/dist/cli/commands/compare.test.d.ts.map +1 -0
  35. package/dist/cli/commands/completion.d.ts +3 -0
  36. package/dist/cli/commands/completion.d.ts.map +1 -0
  37. package/dist/cli/commands/doctor.d.ts +3 -0
  38. package/dist/cli/commands/doctor.d.ts.map +1 -0
  39. package/dist/cli/commands/estimate.d.ts +3 -0
  40. package/dist/cli/commands/estimate.d.ts.map +1 -0
  41. package/dist/cli/commands/generate.d.ts +3 -0
  42. package/dist/cli/commands/generate.d.ts.map +1 -0
  43. package/dist/cli/commands/judge.d.ts +3 -0
  44. package/dist/cli/commands/judge.d.ts.map +1 -0
  45. package/dist/cli/commands/mcp.d.ts +3 -0
  46. package/dist/cli/commands/mcp.d.ts.map +1 -0
  47. package/dist/cli/commands/run.d.ts +3 -0
  48. package/dist/cli/commands/run.d.ts.map +1 -0
  49. package/dist/cli/commands/sync.d.ts +3 -0
  50. package/dist/cli/commands/sync.d.ts.map +1 -0
  51. package/dist/cli/index.d.ts +3 -0
  52. package/dist/cli/index.d.ts.map +1 -0
  53. package/dist/cli/index.js +857 -170
  54. package/dist/core/assertions-coverage.test.d.ts +2 -0
  55. package/dist/core/assertions-coverage.test.d.ts.map +1 -0
  56. package/dist/core/assertions.d.ts +18 -0
  57. package/dist/core/assertions.d.ts.map +1 -0
  58. package/dist/core/assertions.test.d.ts +2 -0
  59. package/dist/core/assertions.test.d.ts.map +1 -0
  60. package/dist/core/e2e.test.d.ts +2 -0
  61. package/dist/core/e2e.test.d.ts.map +1 -0
  62. package/dist/core/judge.d.ts +13 -0
  63. package/dist/core/judge.d.ts.map +1 -0
  64. package/dist/core/judge.test.d.ts +2 -0
  65. package/dist/core/judge.test.d.ts.map +1 -0
  66. package/dist/core/reporter.d.ts +21 -0
  67. package/dist/core/reporter.d.ts.map +1 -0
  68. package/dist/core/reporter.test.d.ts +2 -0
  69. package/dist/core/reporter.test.d.ts.map +1 -0
  70. package/dist/core/runner.d.ts +4 -0
  71. package/dist/core/runner.d.ts.map +1 -0
  72. package/dist/core/runner.test.d.ts +2 -0
  73. package/dist/core/runner.test.d.ts.map +1 -0
  74. package/dist/datasets/loader.d.ts +18 -0
  75. package/dist/datasets/loader.d.ts.map +1 -0
  76. package/dist/datasets/loader.test.d.ts +2 -0
  77. package/dist/datasets/loader.test.d.ts.map +1 -0
  78. package/dist/db/store.d.ts +17 -0
  79. package/dist/db/store.d.ts.map +1 -0
  80. package/dist/db/store.test.d.ts +2 -0
  81. package/dist/db/store.test.d.ts.map +1 -0
  82. package/dist/index.d.ts +8 -0
  83. package/dist/index.d.ts.map +1 -0
  84. package/dist/index.js +24 -4
  85. package/dist/mcp/http.d.ts +13 -0
  86. package/dist/mcp/http.d.ts.map +1 -0
  87. package/dist/mcp/http.test.d.ts +2 -0
  88. package/dist/mcp/http.test.d.ts.map +1 -0
  89. package/dist/mcp/index.d.ts +3 -0
  90. package/dist/mcp/index.d.ts.map +1 -0
  91. package/dist/mcp/index.js +32454 -477
  92. package/dist/mcp/mcp.test.d.ts +2 -0
  93. package/dist/mcp/mcp.test.d.ts.map +1 -0
  94. package/dist/mcp/server.d.ts +5 -0
  95. package/dist/mcp/server.d.ts.map +1 -0
  96. package/dist/server/index.d.ts +3 -0
  97. package/dist/server/index.d.ts.map +1 -0
  98. package/dist/server/index.js +24 -4
  99. package/dist/server/server.test.d.ts +2 -0
  100. package/dist/server/server.test.d.ts.map +1 -0
  101. package/dist/types/index.d.ts +171 -0
  102. package/dist/types/index.d.ts.map +1 -0
  103. package/package.json +3 -2
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=adapters.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"adapters.test.d.ts","sourceRoot":"","sources":["../../src/adapters/adapters.test.ts"],"names":[],"mappings":""}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=anthropic-openai.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"anthropic-openai.test.d.ts","sourceRoot":"","sources":["../../src/adapters/anthropic-openai.test.ts"],"names":[],"mappings":""}
@@ -0,0 +1,4 @@
1
+ import type { AnthropicAdapterConfig, ConversationTurn } from "../types/index.js";
2
+ import type { AdapterResponse } from "./http.js";
3
+ export declare function callAnthropicAdapter(config: AnthropicAdapterConfig, input: string, turns?: ConversationTurn[]): Promise<AdapterResponse>;
4
+ //# sourceMappingURL=anthropic.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"anthropic.d.ts","sourceRoot":"","sources":["../../src/adapters/anthropic.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAClF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAEjD,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,sBAAsB,EAC9B,KAAK,EAAE,MAAM,EACb,KAAK,CAAC,EAAE,gBAAgB,EAAE,GACzB,OAAO,CAAC,eAAe,CAAC,CA+C1B"}
@@ -0,0 +1,4 @@
1
+ import type { CliAdapterConfig } from "../types/index.js";
2
+ import type { AdapterResponse } from "./http.js";
3
+ export declare function callCliAdapter(config: CliAdapterConfig, input: string): Promise<AdapterResponse>;
4
+ //# sourceMappingURL=cli.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../src/adapters/cli.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAEjD,wBAAsB,cAAc,CAClC,MAAM,EAAE,gBAAgB,EACxB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,eAAe,CAAC,CA4C1B"}
@@ -0,0 +1,4 @@
1
+ import type { FunctionAdapterConfig } from "../types/index.js";
2
+ import type { AdapterResponse } from "./http.js";
3
+ export declare function callFunctionAdapter(config: FunctionAdapterConfig, input: string): Promise<AdapterResponse>;
4
+ //# sourceMappingURL=function.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"function.d.ts","sourceRoot":"","sources":["../../src/adapters/function.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAC/D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAEjD,wBAAsB,mBAAmB,CACvC,MAAM,EAAE,qBAAqB,EAC7B,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,eAAe,CAAC,CA2B1B"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=http-cli-coverage.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"http-cli-coverage.test.d.ts","sourceRoot":"","sources":["../../src/adapters/http-cli-coverage.test.ts"],"names":[],"mappings":""}
@@ -0,0 +1,16 @@
1
+ import type { HttpAdapterConfig, ConversationTurn } from "../types/index.js";
2
+ export interface AdapterResponse {
3
+ output: string;
4
+ durationMs: number;
5
+ inputTokens?: number;
6
+ outputTokens?: number;
7
+ costUsd?: number;
8
+ toolCalls?: Array<{
9
+ name: string;
10
+ arguments?: Record<string, unknown>;
11
+ }>;
12
+ statusCode?: number;
13
+ error?: string;
14
+ }
15
+ export declare function callHttpAdapter(config: HttpAdapterConfig, input: string, turns?: ConversationTurn[]): Promise<AdapterResponse>;
16
+ //# sourceMappingURL=http.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../../src/adapters/http.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAE7E,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,CAAC,CAAC;IACzE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AA+BD,wBAAsB,eAAe,CACnC,MAAM,EAAE,iBAAiB,EACzB,KAAK,EAAE,MAAM,EACb,KAAK,CAAC,EAAE,gBAAgB,EAAE,GACzB,OAAO,CAAC,eAAe,CAAC,CAqE1B"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=mcp-adapter.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"mcp-adapter.test.d.ts","sourceRoot":"","sources":["../../src/adapters/mcp-adapter.test.ts"],"names":[],"mappings":""}
@@ -0,0 +1,4 @@
1
+ import type { McpAdapterConfig } from "../types/index.js";
2
+ import type { AdapterResponse } from "./http.js";
3
+ export declare function callMcpAdapter(config: McpAdapterConfig, input: string): Promise<AdapterResponse>;
4
+ //# sourceMappingURL=mcp.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"mcp.d.ts","sourceRoot":"","sources":["../../src/adapters/mcp.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAEjD,wBAAsB,cAAc,CAClC,MAAM,EAAE,gBAAgB,EACxB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,eAAe,CAAC,CAqD1B"}
@@ -0,0 +1,4 @@
1
+ import type { OpenAIAdapterConfig, ConversationTurn } from "../types/index.js";
2
+ import type { AdapterResponse } from "./http.js";
3
+ export declare function callOpenAIAdapter(config: OpenAIAdapterConfig, input: string, turns?: ConversationTurn[]): Promise<AdapterResponse>;
4
+ //# sourceMappingURL=openai.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"openai.d.ts","sourceRoot":"","sources":["../../src/adapters/openai.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAC/E,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAEjD,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,mBAAmB,EAC3B,KAAK,EAAE,MAAM,EACb,KAAK,CAAC,EAAE,gBAAgB,EAAE,GACzB,OAAO,CAAC,eAAe,CAAC,CAuD1B"}
@@ -0,0 +1,3 @@
1
+ import type { AdapterConfig } from "../types/index.js";
2
+ export declare function parseAdapterConfig(opts: Record<string, string>): AdapterConfig;
3
+ //# sourceMappingURL=adapter-parser.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"adapter-parser.d.ts","sourceRoot":"","sources":["../../src/cli/adapter-parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAEvD,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,aAAa,CA8B9E"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=cli.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"cli.test.d.ts","sourceRoot":"","sources":["../../src/cli/cli.test.ts"],"names":[],"mappings":""}
@@ -0,0 +1,3 @@
1
+ import { Command } from "commander";
2
+ export declare function calibrateCommand(): Command;
3
+ //# sourceMappingURL=calibrate.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"calibrate.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/calibrate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,wBAAgB,gBAAgB,IAAI,OAAO,CA+C1C"}
@@ -0,0 +1,3 @@
1
+ import { Command } from "commander";
2
+ export declare function captureCommand(): Command;
3
+ //# sourceMappingURL=capture.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"capture.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/capture.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,wBAAgB,cAAc,IAAI,OAAO,CAyExC"}
@@ -0,0 +1,3 @@
1
+ import { Command } from "commander";
2
+ export declare function ciCommand(): Command;
3
+ //# sourceMappingURL=ci.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ci.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/ci.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,wBAAgB,SAAS,IAAI,OAAO,CAiEnC"}
@@ -0,0 +1,5 @@
1
+ import { Command } from "commander";
2
+ import { compareRuns } from "../../core/reporter.js";
3
+ export declare function renderMarkdownDiff(diff: ReturnType<typeof compareRuns>): string;
4
+ export declare function compareCommand(): Command;
5
+ //# sourceMappingURL=compare.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"compare.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/compare.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,WAAW,EAA+B,MAAM,wBAAwB,CAAC;AAGlF,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,UAAU,CAAC,OAAO,WAAW,CAAC,GAAG,MAAM,CA2B/E;AAED,wBAAgB,cAAc,IAAI,OAAO,CAgCxC"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=compare.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"compare.test.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/compare.test.ts"],"names":[],"mappings":""}
@@ -0,0 +1,3 @@
1
+ import { Command } from "commander";
2
+ export declare function completionCommand(): Command;
3
+ //# sourceMappingURL=completion.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"completion.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/completion.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAiFpC,wBAAgB,iBAAiB,IAAI,OAAO,CAgB3C"}
@@ -0,0 +1,3 @@
1
+ import { Command } from "commander";
2
+ export declare function doctorCommand(): Command;
3
+ //# sourceMappingURL=doctor.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"doctor.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/doctor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAqCpC,wBAAgB,aAAa,IAAI,OAAO,CAqFvC"}
@@ -0,0 +1,3 @@
1
+ import { Command } from "commander";
2
+ export declare function estimateCommand(): Command;
3
+ //# sourceMappingURL=estimate.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"estimate.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/estimate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAapC,wBAAgB,eAAe,IAAI,OAAO,CA2CzC"}
@@ -0,0 +1,3 @@
1
+ import { Command } from "commander";
2
+ export declare function generateCommand(): Command;
3
+ //# sourceMappingURL=generate.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"generate.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/generate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAiBpC,wBAAgB,eAAe,IAAI,OAAO,CAyDzC"}
@@ -0,0 +1,3 @@
1
+ import { Command } from "commander";
2
+ export declare function judgeCommand(): Command;
3
+ //# sourceMappingURL=judge.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/judge.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,wBAAgB,YAAY,IAAI,OAAO,CA+BtC"}
@@ -0,0 +1,3 @@
1
+ import { Command } from "commander";
2
+ export declare function mcpCommand(): Command;
3
+ //# sourceMappingURL=mcp.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"mcp.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/mcp.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,wBAAgB,UAAU,IAAI,OAAO,CAgCpC"}
@@ -0,0 +1,3 @@
1
+ import { Command } from "commander";
2
+ export declare function runCommand(): Command;
3
+ //# sourceMappingURL=run.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,wBAAgB,UAAU,IAAI,OAAO,CAyDpC"}
@@ -0,0 +1,3 @@
1
+ import { Command } from "commander";
2
+ export declare function syncCommand(): Command;
3
+ //# sourceMappingURL=sync.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/sync.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,wBAAgB,WAAW,IAAI,OAAO,CA2FrC"}
@@ -0,0 +1,3 @@
1
+ #!/usr/bin/env bun
2
+ export {};
3
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/cli/index.ts"],"names":[],"mappings":""}