@terreno/ai 57.21.0 → 57.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. package/CHANGELOG.md +31 -0
  2. package/README.md +3 -3
  3. package/biome.jsonc +2 -2
  4. package/dist/harness/commit.d.ts +4 -3
  5. package/dist/harness/commit.js +23 -4
  6. package/dist/index.d.ts +6 -9
  7. package/dist/index.js +14 -25
  8. package/dist/observability/adminScreens.js +5 -1
  9. package/dist/observability/evaluate.d.ts +8 -1
  10. package/dist/observability/evaluate.js +10 -2
  11. package/dist/observability/judgeAgreement.d.ts +53 -0
  12. package/dist/observability/judgeAgreement.js +123 -0
  13. package/dist/observability/local/datasetStore.d.ts +5 -0
  14. package/dist/observability/local/datasetStore.js +7 -2
  15. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  16. package/dist/observability/local/evaluatorStore.js +73 -2
  17. package/dist/observability/local/experimentRunner.d.ts +40 -3
  18. package/dist/observability/local/experimentRunner.js +390 -11
  19. package/dist/observability/local/humanAnswers.d.ts +57 -0
  20. package/dist/observability/local/humanAnswers.js +174 -0
  21. package/dist/observability/local/judgeRun.d.ts +56 -0
  22. package/dist/observability/local/judgeRun.js +111 -0
  23. package/dist/observability/local/localPlugin.js +4 -0
  24. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  25. package/dist/observability/local/models/obsEvaluator.js +4 -0
  26. package/dist/observability/local/models/obsExperiment.js +28 -0
  27. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  28. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  29. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  30. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  31. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  32. package/dist/observability/local/models/obsReviewItem.js +10 -2
  33. package/dist/observability/local/models/obsTrace.js +4 -0
  34. package/dist/observability/local/promptReferences.d.ts +31 -0
  35. package/dist/observability/local/promptReferences.js +114 -0
  36. package/dist/observability/local/promptStore.d.ts +55 -19
  37. package/dist/observability/local/promptStore.js +171 -7
  38. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  39. package/dist/observability/local/reviewBatchStore.js +517 -0
  40. package/dist/observability/local/reviewStore.d.ts +24 -3
  41. package/dist/observability/local/reviewStore.js +98 -22
  42. package/dist/observability/local/traceStore.d.ts +14 -1
  43. package/dist/observability/local/traceStore.js +144 -11
  44. package/dist/observability/observabilityApp.js +10 -1
  45. package/dist/observability/questionFingerprint.d.ts +7 -0
  46. package/dist/observability/questionFingerprint.js +23 -0
  47. package/dist/observability/routes/experiments.js +12 -1
  48. package/dist/observability/routes/review.js +3 -1
  49. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  50. package/dist/observability/routes/reviewBatches.js +78 -0
  51. package/dist/observability/routes/traces.d.ts +1 -1
  52. package/dist/observability/routes/traces.js +34 -2
  53. package/dist/observability/types.d.ts +10 -1
  54. package/dist/service/aiService.js +4 -1
  55. package/dist/service/chatTurn.js +38 -31
  56. package/dist/types/harness.d.ts +5 -2
  57. package/dist/types/index.d.ts +8 -3
  58. package/dist/types/observability.d.ts +74 -2
  59. package/package.json +8 -24
  60. package/src/harness/agentLoop.test.ts +0 -3
  61. package/src/harness/agentRecovery.test.ts +0 -3
  62. package/src/harness/commit.ts +24 -4
  63. package/src/harness/extensions.test.ts +0 -3
  64. package/src/harness/subagent.test.ts +27 -3
  65. package/src/index.test.ts +0 -17
  66. package/src/index.ts +23 -26
  67. package/src/isolated/gptStream.isolated.ts +0 -8
  68. package/src/observability/adminScreens.test.ts +7 -0
  69. package/src/observability/adminScreens.ts +6 -1
  70. package/src/observability/evaluate.ts +12 -2
  71. package/src/observability/judgeAgreement.test.ts +111 -0
  72. package/src/observability/judgeAgreement.ts +184 -0
  73. package/src/observability/local/datasetStore.ts +30 -11
  74. package/src/observability/local/evaluatorStore.ts +96 -2
  75. package/src/observability/local/experimentRunner.ts +495 -15
  76. package/src/observability/local/humanAnswers.ts +233 -0
  77. package/src/observability/local/judgePipeline.test.ts +444 -0
  78. package/src/observability/local/judgeRun.test.ts +173 -0
  79. package/src/observability/local/judgeRun.ts +184 -0
  80. package/src/observability/local/localPlugin.ts +4 -0
  81. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  82. package/src/observability/local/models/obsEvaluator.ts +4 -0
  83. package/src/observability/local/models/obsExperiment.ts +29 -0
  84. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  85. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  86. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  87. package/src/observability/local/models/obsReviewItem.ts +11 -2
  88. package/src/observability/local/models/obsTrace.ts +5 -0
  89. package/src/observability/local/obsModels.test.ts +13 -0
  90. package/src/observability/local/promptReferences.test.ts +128 -0
  91. package/src/observability/local/promptReferences.ts +149 -0
  92. package/src/observability/local/promptStore.test.ts +291 -0
  93. package/src/observability/local/promptStore.ts +243 -32
  94. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  95. package/src/observability/local/reviewBatchStore.ts +656 -0
  96. package/src/observability/local/reviewStore.ts +122 -25
  97. package/src/observability/local/traceStore.test.ts +139 -1
  98. package/src/observability/local/traceStore.ts +162 -11
  99. package/src/observability/observabilityApp.test.ts +13 -10
  100. package/src/observability/observabilityApp.ts +10 -1
  101. package/src/observability/questionFingerprint.test.ts +33 -0
  102. package/src/observability/questionFingerprint.ts +24 -0
  103. package/src/observability/routes/experiments.test.ts +37 -0
  104. package/src/observability/routes/experiments.ts +13 -1
  105. package/src/observability/routes/review.ts +3 -1
  106. package/src/observability/routes/reviewBatches.test.ts +209 -0
  107. package/src/observability/routes/reviewBatches.ts +138 -0
  108. package/src/observability/routes/status.test.ts +4 -4
  109. package/src/observability/routes/traces.test.ts +61 -0
  110. package/src/observability/routes/traces.ts +46 -4
  111. package/src/observability/types.ts +5 -2
  112. package/src/routes/routes.test.ts +101 -76
  113. package/src/service/aiService.test.ts +21 -0
  114. package/src/service/aiService.ts +4 -1
  115. package/src/service/chatTurn.ts +42 -33
  116. package/src/tests/bunSetup.ts +0 -54
  117. package/src/types/harness.ts +5 -2
  118. package/src/types/index.ts +8 -3
  119. package/src/types/observability.ts +86 -2
  120. package/dist/langfuseApp.d.ts +0 -9
  121. package/dist/langfuseApp.js +0 -53
  122. package/dist/langfuseCache.d.ts +0 -15
  123. package/dist/langfuseCache.js +0 -54
  124. package/dist/langfuseClient.d.ts +0 -6
  125. package/dist/langfuseClient.js +0 -34
  126. package/dist/langfusePrompts.d.ts +0 -12
  127. package/dist/langfusePrompts.js +0 -89
  128. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  129. package/dist/langfuseRoutesEvaluations.js +0 -32
  130. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  131. package/dist/langfuseRoutesMiddleware.js +0 -13
  132. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  133. package/dist/langfuseRoutesPlayground.js +0 -36
  134. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  135. package/dist/langfuseRoutesPrompts.js +0 -56
  136. package/dist/langfuseRoutesTraces.d.ts +0 -2
  137. package/dist/langfuseRoutesTraces.js +0 -30
  138. package/dist/langfuseTracing.d.ts +0 -4
  139. package/dist/langfuseTracing.js +0 -32
  140. package/dist/langfuseTypes.d.ts +0 -96
  141. package/dist/langfuseTypes.js +0 -2
  142. package/dist/langfuseVercelAi.d.ts +0 -12
  143. package/dist/langfuseVercelAi.js +0 -49
  144. package/src/isolated/langfuseClient.isolated.ts +0 -66
  145. package/src/langfuseApp.test.ts +0 -119
  146. package/src/langfuseApp.ts +0 -62
  147. package/src/langfuseCache.test.ts +0 -75
  148. package/src/langfuseCache.ts +0 -72
  149. package/src/langfuseClient.test.ts +0 -69
  150. package/src/langfuseClient.ts +0 -34
  151. package/src/langfusePrompts.test.ts +0 -188
  152. package/src/langfusePrompts.ts +0 -121
  153. package/src/langfuseRoutes.test.ts +0 -261
  154. package/src/langfuseRoutesEvaluations.ts +0 -48
  155. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  156. package/src/langfuseRoutesMiddleware.ts +0 -12
  157. package/src/langfuseRoutesPlayground.ts +0 -50
  158. package/src/langfuseRoutesPrompts.ts +0 -87
  159. package/src/langfuseRoutesTraces.ts +0 -40
  160. package/src/langfuseTracing.test.ts +0 -72
  161. package/src/langfuseTracing.ts +0 -33
  162. package/src/langfuseTypes.ts +0 -103
  163. package/src/langfuseVercelAi.test.ts +0 -82
  164. package/src/langfuseVercelAi.ts +0 -72
  165. package/src/tests/fakeLangfusePrompt.ts +0 -10
@@ -4,7 +4,10 @@ exports.LocalEvaluatorStore = void 0;
4
4
  const api_1 = require("@terreno/api");
5
5
  const evaluate_1 = require("../evaluate");
6
6
  const evaluatorTemplates_1 = require("../evaluatorTemplates");
7
+ const questionFingerprint_1 = require("../questionFingerprint");
8
+ const obsDataset_1 = require("./models/obsDataset");
7
9
  const obsEvaluator_1 = require("./models/obsEvaluator");
10
+ const obsExperiment_1 = require("./models/obsExperiment");
8
11
  const promptStore_1 = require("./promptStore");
9
12
  const defaultRunModes = (partial) => {
10
13
  return {
@@ -102,8 +105,9 @@ class LocalEvaluatorStore {
102
105
  }
103
106
  async list() {
104
107
  const rows = await (0, obsEvaluator_1.registerObsEvaluator)().find({}).sort({ name: 1 });
108
+ const calibrations = await this.loadCalibrations(rows);
105
109
  return rows.map((row) => {
106
- return toView(row);
110
+ return { ...toView(row), calibration: calibrations.get(String(row._id)) };
107
111
  });
108
112
  }
109
113
  async get(id) {
@@ -111,7 +115,74 @@ class LocalEvaluatorStore {
111
115
  if (!doc) {
112
116
  throw new api_1.APIError({ status: 404, title: `Unknown evaluator "${id}"` });
113
117
  }
114
- return toView(doc);
118
+ const calibrations = await this.loadCalibrations([doc]);
119
+ return { ...toView(doc), calibration: calibrations.get(String(doc._id)) };
120
+ }
121
+ // Latest completed judge experiment per llm-judge evaluator, summarized for display.
122
+ async loadCalibrations(evaluators) {
123
+ const result = new Map();
124
+ const judges = evaluators.filter((row) => row.type === "llm-judge");
125
+ if (judges.length === 0) {
126
+ return result;
127
+ }
128
+ const experiments = await (0, obsExperiment_1.registerObsExperiment)()
129
+ .find({
130
+ judgeEvaluatorId: { $in: judges.map((row) => row._id) },
131
+ kind: "judge",
132
+ status: "completed",
133
+ })
134
+ .sort({ created: -1 });
135
+ const latest = new Map();
136
+ for (const experiment of experiments) {
137
+ const key = String(experiment.judgeEvaluatorId);
138
+ if (!latest.has(key)) {
139
+ latest.set(key, experiment);
140
+ }
141
+ }
142
+ if (latest.size === 0) {
143
+ return result;
144
+ }
145
+ const rows = [...latest.values()];
146
+ const datasets = await (0, obsDataset_1.registerObsDataset)().find({
147
+ _id: { $in: rows.map((row) => row.datasetId) },
148
+ });
149
+ const questionSets = await (0, obsEvaluator_1.registerObsEvaluator)().find({
150
+ _id: { $in: rows.flatMap((row) => (row.questionSetId ? [row.questionSetId] : [])) },
151
+ });
152
+ for (const judge of judges) {
153
+ const experiment = latest.get(String(judge._id));
154
+ if (!experiment) {
155
+ continue;
156
+ }
157
+ const production = judge.judgePromptName
158
+ ? (await this.promptStore.getVersionByLabel(judge.judgePromptName, "production"))?.version
159
+ : undefined;
160
+ const agreement = experiment.results?.agreement ?? {};
161
+ const best = [...experiment.versions].sort((left, right) => {
162
+ return (agreement[String(right)]?.rate ?? -1) - (agreement[String(left)]?.rate ?? -1);
163
+ })[0];
164
+ const version = production !== undefined && experiment.versions.includes(production) ? production : best;
165
+ if (version === undefined) {
166
+ continue;
167
+ }
168
+ const questionSet = questionSets.find((row) => {
169
+ return String(row._id) === String(experiment.questionSetId);
170
+ });
171
+ const gate = experiment.results?.gates.find((row) => row.version === version);
172
+ result.set(String(judge._id), {
173
+ datasetName: datasets.find((row) => String(row._id) === String(experiment.datasetId))?.name,
174
+ experimentId: String(experiment._id),
175
+ experimentName: experiment.name,
176
+ passed: gate?.passed ?? false,
177
+ questionSetName: questionSet?.name,
178
+ rate: agreement[String(version)]?.rate,
179
+ stale: !questionSet ||
180
+ (0, questionFingerprint_1.questionSetFingerprint)(questionSet.dimensions) !== experiment.questionsFingerprint,
181
+ targetAgreement: experiment.targetAgreement,
182
+ version,
183
+ });
184
+ }
185
+ return result;
115
186
  }
116
187
  async getByName(name) {
117
188
  const doc = await (0, obsEvaluator_1.registerObsEvaluator)().findOneOrNone({ name });
@@ -1,4 +1,4 @@
1
- import type { ExperimentAggregates, ExperimentEstimate, ExperimentVersionResult, ScoreThreshold } from "../../types/observability";
1
+ import type { ExperimentAggregates, ExperimentEstimate, ExperimentKind, ExperimentVersionResult, ScoreThreshold } from "../../types/observability";
2
2
  import type { ModelPrice, ObservabilityAiServiceFactory, ObservabilityGenerateClient } from "../types";
3
3
  import type { LocalDatasetStore } from "./datasetStore";
4
4
  import type { LocalEvaluatorStore } from "./evaluatorStore";
@@ -7,26 +7,54 @@ export interface ExperimentCreateInput {
7
7
  datasetId: string;
8
8
  evaluatorIds: string[];
9
9
  includeUnproofread?: boolean;
10
+ kind?: "prompt";
10
11
  modelOverride?: string;
11
12
  name: string;
12
13
  promptName: string;
13
14
  thresholds?: ScoreThreshold[];
14
15
  versions: number[];
15
16
  }
17
+ /**
18
+ * Judge calibration: run 2–3 versions of an llm-judge's prompt over dataset items humans already
19
+ * answered with `questionSetId`, and rank versions by agreement with those answers.
20
+ */
21
+ export interface JudgeExperimentCreateInput {
22
+ datasetId: string;
23
+ judgeEvaluatorId: string;
24
+ kind: "judge";
25
+ modelOverride?: string;
26
+ name: string;
27
+ questionSetId: string;
28
+ /** Agreement rate (0–1) a version must reach to pass its gate. Defaults to 0.9. */
29
+ targetAgreement?: number;
30
+ /** Versions of the judge evaluator's `judgePromptName`. */
31
+ versions: number[];
32
+ }
33
+ export declare const DEFAULT_TARGET_AGREEMENT = 0.9;
16
34
  export interface ExperimentView {
17
35
  backgroundTaskId?: string;
18
36
  created: string;
19
37
  datasetId: string;
38
+ datasetName?: string;
20
39
  estimate?: ExperimentEstimate;
21
40
  evaluatorIds: string[];
22
41
  id: string;
23
42
  includeUnproofread: boolean;
24
43
  items: ExperimentItemView[];
44
+ judgeEvaluatorId?: string;
45
+ judgeName?: string;
46
+ kind: ExperimentKind;
25
47
  modelOverride?: string;
26
48
  name: string;
27
49
  promptName: string;
50
+ questionSetId?: string;
51
+ questionSetName?: string;
52
+ questionsFingerprint?: string;
53
+ /** Judge experiments: true when the question set's questions changed since the run. */
54
+ questionsStale?: boolean;
28
55
  results?: ExperimentAggregates;
29
56
  status: "completed" | "failed" | "pending" | "running";
57
+ targetAgreement?: number;
30
58
  thresholds: ScoreThreshold[];
31
59
  updated: string;
32
60
  versions: number[];
@@ -34,6 +62,8 @@ export interface ExperimentView {
34
62
  export interface ExperimentItemView {
35
63
  datasetItemId: string;
36
64
  failed: boolean;
65
+ /** Judge experiments: the human answers each judge version is compared with. */
66
+ humanScores?: Record<string, boolean | number | string>;
37
67
  id: string;
38
68
  versionResults: Record<string, ExperimentVersionResult>;
39
69
  }
@@ -73,21 +103,28 @@ export declare class LocalExperimentRunner {
73
103
  private resolveAiClient;
74
104
  estimate(input: {
75
105
  datasetId: string;
76
- evaluatorIds: string[];
106
+ evaluatorIds?: string[];
77
107
  includeUnproofread?: boolean;
108
+ kind?: ExperimentKind;
78
109
  modelOverride?: string;
110
+ questionSetId?: string;
79
111
  versions: number[];
80
112
  }): Promise<ExperimentEstimate>;
81
113
  private assertAutomaticEvaluators;
82
- create(input: ExperimentCreateInput): Promise<ExperimentView>;
114
+ create(input: ExperimentCreateInput | JudgeExperimentCreateInput): Promise<ExperimentView>;
115
+ private createJudge;
116
+ private startRun;
83
117
  list(params?: {
118
+ kind?: ExperimentKind;
84
119
  limit?: number;
85
120
  page?: number;
86
121
  promptName?: string;
87
122
  }): Promise<ExperimentListResult>;
88
123
  get(id: string): Promise<ExperimentView>;
124
+ private loadNames;
89
125
  private toSummaryView;
90
126
  promote(id: string, version: number): Promise<MoveLabelResult>;
91
127
  executeExperiment(id: string): Promise<void>;
128
+ private executeJudgeExperiment;
92
129
  private computeGates;
93
130
  }