@terreno/ai 58.0.0-beta.2 → 58.0.0-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. package/CHANGELOG.md +146 -0
  2. package/README.md +3 -3
  3. package/dist/harness/commit.d.ts +4 -3
  4. package/dist/harness/commit.js +23 -4
  5. package/dist/index.d.ts +6 -9
  6. package/dist/index.js +14 -25
  7. package/dist/observability/adminScreens.js +5 -1
  8. package/dist/observability/evaluate.d.ts +8 -1
  9. package/dist/observability/evaluate.js +10 -2
  10. package/dist/observability/judgeAgreement.d.ts +53 -0
  11. package/dist/observability/judgeAgreement.js +123 -0
  12. package/dist/observability/local/datasetStore.d.ts +5 -0
  13. package/dist/observability/local/datasetStore.js +7 -2
  14. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  15. package/dist/observability/local/evaluatorStore.js +73 -2
  16. package/dist/observability/local/experimentRunner.d.ts +40 -3
  17. package/dist/observability/local/experimentRunner.js +390 -11
  18. package/dist/observability/local/humanAnswers.d.ts +57 -0
  19. package/dist/observability/local/humanAnswers.js +174 -0
  20. package/dist/observability/local/judgeRun.d.ts +56 -0
  21. package/dist/observability/local/judgeRun.js +111 -0
  22. package/dist/observability/local/localPlugin.js +4 -0
  23. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  24. package/dist/observability/local/models/obsEvaluator.js +4 -0
  25. package/dist/observability/local/models/obsExperiment.js +28 -0
  26. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  27. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  28. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  29. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  30. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  31. package/dist/observability/local/models/obsReviewItem.js +10 -2
  32. package/dist/observability/local/models/obsTrace.js +4 -0
  33. package/dist/observability/local/promptReferences.d.ts +31 -0
  34. package/dist/observability/local/promptReferences.js +114 -0
  35. package/dist/observability/local/promptStore.d.ts +55 -19
  36. package/dist/observability/local/promptStore.js +171 -7
  37. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  38. package/dist/observability/local/reviewBatchStore.js +517 -0
  39. package/dist/observability/local/reviewStore.d.ts +24 -3
  40. package/dist/observability/local/reviewStore.js +98 -22
  41. package/dist/observability/local/traceStore.d.ts +14 -1
  42. package/dist/observability/local/traceStore.js +144 -11
  43. package/dist/observability/observabilityApp.js +10 -1
  44. package/dist/observability/questionFingerprint.d.ts +7 -0
  45. package/dist/observability/questionFingerprint.js +23 -0
  46. package/dist/observability/routes/experiments.js +12 -1
  47. package/dist/observability/routes/review.js +3 -1
  48. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  49. package/dist/observability/routes/reviewBatches.js +78 -0
  50. package/dist/observability/routes/traces.d.ts +1 -1
  51. package/dist/observability/routes/traces.js +34 -2
  52. package/dist/observability/types.d.ts +10 -1
  53. package/dist/service/aiService.js +4 -1
  54. package/dist/service/chatTurn.js +38 -31
  55. package/dist/types/harness.d.ts +5 -2
  56. package/dist/types/index.d.ts +8 -3
  57. package/dist/types/observability.d.ts +74 -2
  58. package/package.json +8 -24
  59. package/src/harness/agentLoop.test.ts +0 -3
  60. package/src/harness/agentRecovery.test.ts +0 -3
  61. package/src/harness/commit.ts +24 -4
  62. package/src/harness/extensions.test.ts +0 -3
  63. package/src/harness/subagent.test.ts +27 -3
  64. package/src/index.test.ts +0 -17
  65. package/src/index.ts +23 -26
  66. package/src/isolated/gptStream.isolated.ts +0 -8
  67. package/src/observability/adminScreens.test.ts +7 -0
  68. package/src/observability/adminScreens.ts +6 -1
  69. package/src/observability/evaluate.ts +12 -2
  70. package/src/observability/judgeAgreement.test.ts +111 -0
  71. package/src/observability/judgeAgreement.ts +184 -0
  72. package/src/observability/local/datasetStore.ts +30 -11
  73. package/src/observability/local/evaluatorStore.ts +96 -2
  74. package/src/observability/local/experimentRunner.ts +495 -15
  75. package/src/observability/local/humanAnswers.ts +233 -0
  76. package/src/observability/local/judgePipeline.test.ts +444 -0
  77. package/src/observability/local/judgeRun.test.ts +173 -0
  78. package/src/observability/local/judgeRun.ts +184 -0
  79. package/src/observability/local/localPlugin.ts +4 -0
  80. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  81. package/src/observability/local/models/obsEvaluator.ts +4 -0
  82. package/src/observability/local/models/obsExperiment.ts +29 -0
  83. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  84. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  85. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  86. package/src/observability/local/models/obsReviewItem.ts +11 -2
  87. package/src/observability/local/models/obsTrace.ts +5 -0
  88. package/src/observability/local/obsModels.test.ts +13 -0
  89. package/src/observability/local/promptReferences.test.ts +128 -0
  90. package/src/observability/local/promptReferences.ts +149 -0
  91. package/src/observability/local/promptStore.test.ts +291 -0
  92. package/src/observability/local/promptStore.ts +243 -32
  93. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  94. package/src/observability/local/reviewBatchStore.ts +656 -0
  95. package/src/observability/local/reviewStore.ts +122 -25
  96. package/src/observability/local/traceStore.test.ts +139 -1
  97. package/src/observability/local/traceStore.ts +162 -11
  98. package/src/observability/observabilityApp.test.ts +13 -10
  99. package/src/observability/observabilityApp.ts +10 -1
  100. package/src/observability/questionFingerprint.test.ts +33 -0
  101. package/src/observability/questionFingerprint.ts +24 -0
  102. package/src/observability/routes/experiments.test.ts +37 -0
  103. package/src/observability/routes/experiments.ts +13 -1
  104. package/src/observability/routes/review.ts +3 -1
  105. package/src/observability/routes/reviewBatches.test.ts +209 -0
  106. package/src/observability/routes/reviewBatches.ts +138 -0
  107. package/src/observability/routes/status.test.ts +4 -4
  108. package/src/observability/routes/traces.test.ts +61 -0
  109. package/src/observability/routes/traces.ts +46 -4
  110. package/src/observability/types.ts +5 -2
  111. package/src/routes/routes.test.ts +101 -76
  112. package/src/service/aiService.test.ts +21 -0
  113. package/src/service/aiService.ts +4 -1
  114. package/src/service/chatTurn.ts +42 -33
  115. package/src/tests/bunSetup.ts +0 -54
  116. package/src/types/harness.ts +5 -2
  117. package/src/types/index.ts +8 -3
  118. package/src/types/observability.ts +86 -2
  119. package/dist/langfuseApp.d.ts +0 -9
  120. package/dist/langfuseApp.js +0 -53
  121. package/dist/langfuseCache.d.ts +0 -15
  122. package/dist/langfuseCache.js +0 -54
  123. package/dist/langfuseClient.d.ts +0 -6
  124. package/dist/langfuseClient.js +0 -34
  125. package/dist/langfusePrompts.d.ts +0 -12
  126. package/dist/langfusePrompts.js +0 -89
  127. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  128. package/dist/langfuseRoutesEvaluations.js +0 -32
  129. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  130. package/dist/langfuseRoutesMiddleware.js +0 -13
  131. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  132. package/dist/langfuseRoutesPlayground.js +0 -36
  133. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  134. package/dist/langfuseRoutesPrompts.js +0 -56
  135. package/dist/langfuseRoutesTraces.d.ts +0 -2
  136. package/dist/langfuseRoutesTraces.js +0 -30
  137. package/dist/langfuseTracing.d.ts +0 -4
  138. package/dist/langfuseTracing.js +0 -32
  139. package/dist/langfuseTypes.d.ts +0 -96
  140. package/dist/langfuseTypes.js +0 -2
  141. package/dist/langfuseVercelAi.d.ts +0 -12
  142. package/dist/langfuseVercelAi.js +0 -49
  143. package/src/isolated/langfuseClient.isolated.ts +0 -66
  144. package/src/langfuseApp.test.ts +0 -119
  145. package/src/langfuseApp.ts +0 -62
  146. package/src/langfuseCache.test.ts +0 -75
  147. package/src/langfuseCache.ts +0 -72
  148. package/src/langfuseClient.test.ts +0 -69
  149. package/src/langfuseClient.ts +0 -34
  150. package/src/langfusePrompts.test.ts +0 -188
  151. package/src/langfusePrompts.ts +0 -121
  152. package/src/langfuseRoutes.test.ts +0 -261
  153. package/src/langfuseRoutesEvaluations.ts +0 -48
  154. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  155. package/src/langfuseRoutesMiddleware.ts +0 -12
  156. package/src/langfuseRoutesPlayground.ts +0 -50
  157. package/src/langfuseRoutesPrompts.ts +0 -87
  158. package/src/langfuseRoutesTraces.ts +0 -40
  159. package/src/langfuseTracing.test.ts +0 -72
  160. package/src/langfuseTracing.ts +0 -33
  161. package/src/langfuseTypes.ts +0 -103
  162. package/src/langfuseVercelAi.test.ts +0 -82
  163. package/src/langfuseVercelAi.ts +0 -72
  164. package/src/tests/fakeLangfusePrompt.ts +0 -10
@@ -4,10 +4,14 @@ import type {
4
4
  EvaluatorDimension,
5
5
  EvaluatorRunModes,
6
6
  ObsEvaluatorDocument,
7
+ ObsExperimentDocument,
7
8
  } from "../../types/observability";
8
9
  import {judgeSchemaMissingDimensions} from "../evaluate";
9
10
  import {getEvaluatorTemplate} from "../evaluatorTemplates";
11
+ import {questionSetFingerprint} from "../questionFingerprint";
12
+ import {registerObsDataset} from "./models/obsDataset";
10
13
  import {registerObsEvaluator} from "./models/obsEvaluator";
14
+ import {registerObsExperiment} from "./models/obsExperiment";
11
15
  import {LocalPromptStore} from "./promptStore";
12
16
 
13
17
  export interface EvaluatorWriteInput {
@@ -23,8 +27,25 @@ export interface EvaluatorWriteInput {
23
27
  type: "human" | "json-assert" | "llm-judge";
24
28
  }
25
29
 
30
+ /** How well an llm-judge agreed with human answers in its latest judge calibration experiment. */
31
+ export interface EvaluatorCalibration {
32
+ datasetName?: string;
33
+ experimentId: string;
34
+ experimentName: string;
35
+ passed: boolean;
36
+ questionSetName?: string;
37
+ /** Agreement rate (0–1) of `version`; unset when nothing could be compared. */
38
+ rate?: number;
39
+ /** True when the question set's questions changed since the experiment ran. */
40
+ stale: boolean;
41
+ targetAgreement?: number;
42
+ /** The judge prompt's production version when it was compared, else the best version. */
43
+ version: number;
44
+ }
45
+
26
46
  export interface EvaluatorView {
27
47
  assertion?: {constraint: string; path: string};
48
+ calibration?: EvaluatorCalibration;
28
49
  confidenceAlertBelow: number;
29
50
  description?: string;
30
51
  dimensions: EvaluatorDimension[];
@@ -140,8 +161,9 @@ export class LocalEvaluatorStore {
140
161
 
141
162
  async list(): Promise<EvaluatorView[]> {
142
163
  const rows = await registerObsEvaluator().find({}).sort({name: 1});
164
+ const calibrations = await this.loadCalibrations(rows);
143
165
  return rows.map((row) => {
144
- return toView(row);
166
+ return {...toView(row), calibration: calibrations.get(String(row._id))};
145
167
  });
146
168
  }
147
169
 
@@ -150,7 +172,79 @@ export class LocalEvaluatorStore {
150
172
  if (!doc) {
151
173
  throw new APIError({status: 404, title: `Unknown evaluator "${id}"`});
152
174
  }
153
- return toView(doc);
175
+ const calibrations = await this.loadCalibrations([doc]);
176
+ return {...toView(doc), calibration: calibrations.get(String(doc._id))};
177
+ }
178
+
179
+ // Latest completed judge experiment per llm-judge evaluator, summarized for display.
180
+ private async loadCalibrations(
181
+ evaluators: ObsEvaluatorDocument[]
182
+ ): Promise<Map<string, EvaluatorCalibration>> {
183
+ const result = new Map<string, EvaluatorCalibration>();
184
+ const judges = evaluators.filter((row) => row.type === "llm-judge");
185
+ if (judges.length === 0) {
186
+ return result;
187
+ }
188
+ const experiments = await registerObsExperiment()
189
+ .find({
190
+ judgeEvaluatorId: {$in: judges.map((row) => row._id)},
191
+ kind: "judge",
192
+ status: "completed",
193
+ })
194
+ .sort({created: -1});
195
+ const latest = new Map<string, ObsExperimentDocument>();
196
+ for (const experiment of experiments) {
197
+ const key = String(experiment.judgeEvaluatorId);
198
+ if (!latest.has(key)) {
199
+ latest.set(key, experiment);
200
+ }
201
+ }
202
+ if (latest.size === 0) {
203
+ return result;
204
+ }
205
+ const rows = [...latest.values()];
206
+ const datasets = await registerObsDataset().find({
207
+ _id: {$in: rows.map((row) => row.datasetId)},
208
+ });
209
+ const questionSets = await registerObsEvaluator().find({
210
+ _id: {$in: rows.flatMap((row) => (row.questionSetId ? [row.questionSetId] : []))},
211
+ });
212
+ for (const judge of judges) {
213
+ const experiment = latest.get(String(judge._id));
214
+ if (!experiment) {
215
+ continue;
216
+ }
217
+ const production = judge.judgePromptName
218
+ ? (await this.promptStore.getVersionByLabel(judge.judgePromptName, "production"))?.version
219
+ : undefined;
220
+ const agreement = experiment.results?.agreement ?? {};
221
+ const best = [...experiment.versions].sort((left, right) => {
222
+ return (agreement[String(right)]?.rate ?? -1) - (agreement[String(left)]?.rate ?? -1);
223
+ })[0];
224
+ const version =
225
+ production !== undefined && experiment.versions.includes(production) ? production : best;
226
+ if (version === undefined) {
227
+ continue;
228
+ }
229
+ const questionSet = questionSets.find((row) => {
230
+ return String(row._id) === String(experiment.questionSetId);
231
+ });
232
+ const gate = experiment.results?.gates.find((row) => row.version === version);
233
+ result.set(String(judge._id), {
234
+ datasetName: datasets.find((row) => String(row._id) === String(experiment.datasetId))?.name,
235
+ experimentId: String(experiment._id),
236
+ experimentName: experiment.name,
237
+ passed: gate?.passed ?? false,
238
+ questionSetName: questionSet?.name,
239
+ rate: agreement[String(version)]?.rate,
240
+ stale:
241
+ !questionSet ||
242
+ questionSetFingerprint(questionSet.dimensions) !== experiment.questionsFingerprint,
243
+ targetAgreement: experiment.targetAgreement,
244
+ version,
245
+ });
246
+ }
247
+ return result;
154
248
  }
155
249
 
156
250
  async getByName(name: string): Promise<EvaluatorView | undefined> {