@terreno/ai 57.21.0 → 57.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/README.md +3 -3
- package/biome.jsonc +2 -2
- package/dist/harness/commit.d.ts +4 -3
- package/dist/harness/commit.js +23 -4
- package/dist/index.d.ts +6 -9
- package/dist/index.js +14 -25
- package/dist/observability/adminScreens.js +5 -1
- package/dist/observability/evaluate.d.ts +8 -1
- package/dist/observability/evaluate.js +10 -2
- package/dist/observability/judgeAgreement.d.ts +53 -0
- package/dist/observability/judgeAgreement.js +123 -0
- package/dist/observability/local/datasetStore.d.ts +5 -0
- package/dist/observability/local/datasetStore.js +7 -2
- package/dist/observability/local/evaluatorStore.d.ts +17 -0
- package/dist/observability/local/evaluatorStore.js +73 -2
- package/dist/observability/local/experimentRunner.d.ts +40 -3
- package/dist/observability/local/experimentRunner.js +390 -11
- package/dist/observability/local/humanAnswers.d.ts +57 -0
- package/dist/observability/local/humanAnswers.js +174 -0
- package/dist/observability/local/judgeRun.d.ts +56 -0
- package/dist/observability/local/judgeRun.js +111 -0
- package/dist/observability/local/localPlugin.js +4 -0
- package/dist/observability/local/models/obsDatasetItem.js +4 -0
- package/dist/observability/local/models/obsEvaluator.js +4 -0
- package/dist/observability/local/models/obsExperiment.js +28 -0
- package/dist/observability/local/models/obsExperimentItem.js +4 -0
- package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
- package/dist/observability/local/models/obsJudgeRun.js +60 -0
- package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
- package/dist/observability/local/models/obsReviewBatch.js +48 -0
- package/dist/observability/local/models/obsReviewItem.js +10 -2
- package/dist/observability/local/models/obsTrace.js +4 -0
- package/dist/observability/local/promptReferences.d.ts +31 -0
- package/dist/observability/local/promptReferences.js +114 -0
- package/dist/observability/local/promptStore.d.ts +55 -19
- package/dist/observability/local/promptStore.js +171 -7
- package/dist/observability/local/reviewBatchStore.d.ts +106 -0
- package/dist/observability/local/reviewBatchStore.js +517 -0
- package/dist/observability/local/reviewStore.d.ts +24 -3
- package/dist/observability/local/reviewStore.js +98 -22
- package/dist/observability/local/traceStore.d.ts +14 -1
- package/dist/observability/local/traceStore.js +144 -11
- package/dist/observability/observabilityApp.js +10 -1
- package/dist/observability/questionFingerprint.d.ts +7 -0
- package/dist/observability/questionFingerprint.js +23 -0
- package/dist/observability/routes/experiments.js +12 -1
- package/dist/observability/routes/review.js +3 -1
- package/dist/observability/routes/reviewBatches.d.ts +11 -0
- package/dist/observability/routes/reviewBatches.js +78 -0
- package/dist/observability/routes/traces.d.ts +1 -1
- package/dist/observability/routes/traces.js +34 -2
- package/dist/observability/types.d.ts +10 -1
- package/dist/service/aiService.js +4 -1
- package/dist/service/chatTurn.js +38 -31
- package/dist/types/harness.d.ts +5 -2
- package/dist/types/index.d.ts +8 -3
- package/dist/types/observability.d.ts +74 -2
- package/package.json +8 -24
- package/src/harness/agentLoop.test.ts +0 -3
- package/src/harness/agentRecovery.test.ts +0 -3
- package/src/harness/commit.ts +24 -4
- package/src/harness/extensions.test.ts +0 -3
- package/src/harness/subagent.test.ts +27 -3
- package/src/index.test.ts +0 -17
- package/src/index.ts +23 -26
- package/src/isolated/gptStream.isolated.ts +0 -8
- package/src/observability/adminScreens.test.ts +7 -0
- package/src/observability/adminScreens.ts +6 -1
- package/src/observability/evaluate.ts +12 -2
- package/src/observability/judgeAgreement.test.ts +111 -0
- package/src/observability/judgeAgreement.ts +184 -0
- package/src/observability/local/datasetStore.ts +30 -11
- package/src/observability/local/evaluatorStore.ts +96 -2
- package/src/observability/local/experimentRunner.ts +495 -15
- package/src/observability/local/humanAnswers.ts +233 -0
- package/src/observability/local/judgePipeline.test.ts +444 -0
- package/src/observability/local/judgeRun.test.ts +173 -0
- package/src/observability/local/judgeRun.ts +184 -0
- package/src/observability/local/localPlugin.ts +4 -0
- package/src/observability/local/models/obsDatasetItem.ts +5 -0
- package/src/observability/local/models/obsEvaluator.ts +4 -0
- package/src/observability/local/models/obsExperiment.ts +29 -0
- package/src/observability/local/models/obsExperimentItem.ts +4 -0
- package/src/observability/local/models/obsJudgeRun.ts +61 -0
- package/src/observability/local/models/obsReviewBatch.ts +52 -0
- package/src/observability/local/models/obsReviewItem.ts +11 -2
- package/src/observability/local/models/obsTrace.ts +5 -0
- package/src/observability/local/obsModels.test.ts +13 -0
- package/src/observability/local/promptReferences.test.ts +128 -0
- package/src/observability/local/promptReferences.ts +149 -0
- package/src/observability/local/promptStore.test.ts +291 -0
- package/src/observability/local/promptStore.ts +243 -32
- package/src/observability/local/reviewBatchStore.test.ts +251 -0
- package/src/observability/local/reviewBatchStore.ts +656 -0
- package/src/observability/local/reviewStore.ts +122 -25
- package/src/observability/local/traceStore.test.ts +139 -1
- package/src/observability/local/traceStore.ts +162 -11
- package/src/observability/observabilityApp.test.ts +13 -10
- package/src/observability/observabilityApp.ts +10 -1
- package/src/observability/questionFingerprint.test.ts +33 -0
- package/src/observability/questionFingerprint.ts +24 -0
- package/src/observability/routes/experiments.test.ts +37 -0
- package/src/observability/routes/experiments.ts +13 -1
- package/src/observability/routes/review.ts +3 -1
- package/src/observability/routes/reviewBatches.test.ts +209 -0
- package/src/observability/routes/reviewBatches.ts +138 -0
- package/src/observability/routes/status.test.ts +4 -4
- package/src/observability/routes/traces.test.ts +61 -0
- package/src/observability/routes/traces.ts +46 -4
- package/src/observability/types.ts +5 -2
- package/src/routes/routes.test.ts +101 -76
- package/src/service/aiService.test.ts +21 -0
- package/src/service/aiService.ts +4 -1
- package/src/service/chatTurn.ts +42 -33
- package/src/tests/bunSetup.ts +0 -54
- package/src/types/harness.ts +5 -2
- package/src/types/index.ts +8 -3
- package/src/types/observability.ts +86 -2
- package/dist/langfuseApp.d.ts +0 -9
- package/dist/langfuseApp.js +0 -53
- package/dist/langfuseCache.d.ts +0 -15
- package/dist/langfuseCache.js +0 -54
- package/dist/langfuseClient.d.ts +0 -6
- package/dist/langfuseClient.js +0 -34
- package/dist/langfusePrompts.d.ts +0 -12
- package/dist/langfusePrompts.js +0 -89
- package/dist/langfuseRoutesEvaluations.d.ts +0 -3
- package/dist/langfuseRoutesEvaluations.js +0 -32
- package/dist/langfuseRoutesMiddleware.d.ts +0 -2
- package/dist/langfuseRoutesMiddleware.js +0 -13
- package/dist/langfuseRoutesPlayground.d.ts +0 -2
- package/dist/langfuseRoutesPlayground.js +0 -36
- package/dist/langfuseRoutesPrompts.d.ts +0 -2
- package/dist/langfuseRoutesPrompts.js +0 -56
- package/dist/langfuseRoutesTraces.d.ts +0 -2
- package/dist/langfuseRoutesTraces.js +0 -30
- package/dist/langfuseTracing.d.ts +0 -4
- package/dist/langfuseTracing.js +0 -32
- package/dist/langfuseTypes.d.ts +0 -96
- package/dist/langfuseTypes.js +0 -2
- package/dist/langfuseVercelAi.d.ts +0 -12
- package/dist/langfuseVercelAi.js +0 -49
- package/src/isolated/langfuseClient.isolated.ts +0 -66
- package/src/langfuseApp.test.ts +0 -119
- package/src/langfuseApp.ts +0 -62
- package/src/langfuseCache.test.ts +0 -75
- package/src/langfuseCache.ts +0 -72
- package/src/langfuseClient.test.ts +0 -69
- package/src/langfuseClient.ts +0 -34
- package/src/langfusePrompts.test.ts +0 -188
- package/src/langfusePrompts.ts +0 -121
- package/src/langfuseRoutes.test.ts +0 -261
- package/src/langfuseRoutesEvaluations.ts +0 -48
- package/src/langfuseRoutesMiddleware.test.ts +0 -57
- package/src/langfuseRoutesMiddleware.ts +0 -12
- package/src/langfuseRoutesPlayground.ts +0 -50
- package/src/langfuseRoutesPrompts.ts +0 -87
- package/src/langfuseRoutesTraces.ts +0 -40
- package/src/langfuseTracing.test.ts +0 -72
- package/src/langfuseTracing.ts +0 -33
- package/src/langfuseTypes.ts +0 -103
- package/src/langfuseVercelAi.test.ts +0 -82
- package/src/langfuseVercelAi.ts +0 -72
- package/src/tests/fakeLangfusePrompt.ts +0 -10
|
@@ -4,10 +4,14 @@ import type {
|
|
|
4
4
|
EvaluatorDimension,
|
|
5
5
|
EvaluatorRunModes,
|
|
6
6
|
ObsEvaluatorDocument,
|
|
7
|
+
ObsExperimentDocument,
|
|
7
8
|
} from "../../types/observability";
|
|
8
9
|
import {judgeSchemaMissingDimensions} from "../evaluate";
|
|
9
10
|
import {getEvaluatorTemplate} from "../evaluatorTemplates";
|
|
11
|
+
import {questionSetFingerprint} from "../questionFingerprint";
|
|
12
|
+
import {registerObsDataset} from "./models/obsDataset";
|
|
10
13
|
import {registerObsEvaluator} from "./models/obsEvaluator";
|
|
14
|
+
import {registerObsExperiment} from "./models/obsExperiment";
|
|
11
15
|
import {LocalPromptStore} from "./promptStore";
|
|
12
16
|
|
|
13
17
|
export interface EvaluatorWriteInput {
|
|
@@ -23,8 +27,25 @@ export interface EvaluatorWriteInput {
|
|
|
23
27
|
type: "human" | "json-assert" | "llm-judge";
|
|
24
28
|
}
|
|
25
29
|
|
|
30
|
+
/** How well an llm-judge agreed with human answers in its latest judge calibration experiment. */
|
|
31
|
+
export interface EvaluatorCalibration {
|
|
32
|
+
datasetName?: string;
|
|
33
|
+
experimentId: string;
|
|
34
|
+
experimentName: string;
|
|
35
|
+
passed: boolean;
|
|
36
|
+
questionSetName?: string;
|
|
37
|
+
/** Agreement rate (0–1) of `version`; unset when nothing could be compared. */
|
|
38
|
+
rate?: number;
|
|
39
|
+
/** True when the question set's questions changed since the experiment ran. */
|
|
40
|
+
stale: boolean;
|
|
41
|
+
targetAgreement?: number;
|
|
42
|
+
/** The judge prompt's production version when it was compared, else the best version. */
|
|
43
|
+
version: number;
|
|
44
|
+
}
|
|
45
|
+
|
|
26
46
|
export interface EvaluatorView {
|
|
27
47
|
assertion?: {constraint: string; path: string};
|
|
48
|
+
calibration?: EvaluatorCalibration;
|
|
28
49
|
confidenceAlertBelow: number;
|
|
29
50
|
description?: string;
|
|
30
51
|
dimensions: EvaluatorDimension[];
|
|
@@ -140,8 +161,9 @@ export class LocalEvaluatorStore {
|
|
|
140
161
|
|
|
141
162
|
async list(): Promise<EvaluatorView[]> {
|
|
142
163
|
const rows = await registerObsEvaluator().find({}).sort({name: 1});
|
|
164
|
+
const calibrations = await this.loadCalibrations(rows);
|
|
143
165
|
return rows.map((row) => {
|
|
144
|
-
return toView(row);
|
|
166
|
+
return {...toView(row), calibration: calibrations.get(String(row._id))};
|
|
145
167
|
});
|
|
146
168
|
}
|
|
147
169
|
|
|
@@ -150,7 +172,79 @@ export class LocalEvaluatorStore {
|
|
|
150
172
|
if (!doc) {
|
|
151
173
|
throw new APIError({status: 404, title: `Unknown evaluator "${id}"`});
|
|
152
174
|
}
|
|
153
|
-
|
|
175
|
+
const calibrations = await this.loadCalibrations([doc]);
|
|
176
|
+
return {...toView(doc), calibration: calibrations.get(String(doc._id))};
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// Latest completed judge experiment per llm-judge evaluator, summarized for display.
|
|
180
|
+
private async loadCalibrations(
|
|
181
|
+
evaluators: ObsEvaluatorDocument[]
|
|
182
|
+
): Promise<Map<string, EvaluatorCalibration>> {
|
|
183
|
+
const result = new Map<string, EvaluatorCalibration>();
|
|
184
|
+
const judges = evaluators.filter((row) => row.type === "llm-judge");
|
|
185
|
+
if (judges.length === 0) {
|
|
186
|
+
return result;
|
|
187
|
+
}
|
|
188
|
+
const experiments = await registerObsExperiment()
|
|
189
|
+
.find({
|
|
190
|
+
judgeEvaluatorId: {$in: judges.map((row) => row._id)},
|
|
191
|
+
kind: "judge",
|
|
192
|
+
status: "completed",
|
|
193
|
+
})
|
|
194
|
+
.sort({created: -1});
|
|
195
|
+
const latest = new Map<string, ObsExperimentDocument>();
|
|
196
|
+
for (const experiment of experiments) {
|
|
197
|
+
const key = String(experiment.judgeEvaluatorId);
|
|
198
|
+
if (!latest.has(key)) {
|
|
199
|
+
latest.set(key, experiment);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
if (latest.size === 0) {
|
|
203
|
+
return result;
|
|
204
|
+
}
|
|
205
|
+
const rows = [...latest.values()];
|
|
206
|
+
const datasets = await registerObsDataset().find({
|
|
207
|
+
_id: {$in: rows.map((row) => row.datasetId)},
|
|
208
|
+
});
|
|
209
|
+
const questionSets = await registerObsEvaluator().find({
|
|
210
|
+
_id: {$in: rows.flatMap((row) => (row.questionSetId ? [row.questionSetId] : []))},
|
|
211
|
+
});
|
|
212
|
+
for (const judge of judges) {
|
|
213
|
+
const experiment = latest.get(String(judge._id));
|
|
214
|
+
if (!experiment) {
|
|
215
|
+
continue;
|
|
216
|
+
}
|
|
217
|
+
const production = judge.judgePromptName
|
|
218
|
+
? (await this.promptStore.getVersionByLabel(judge.judgePromptName, "production"))?.version
|
|
219
|
+
: undefined;
|
|
220
|
+
const agreement = experiment.results?.agreement ?? {};
|
|
221
|
+
const best = [...experiment.versions].sort((left, right) => {
|
|
222
|
+
return (agreement[String(right)]?.rate ?? -1) - (agreement[String(left)]?.rate ?? -1);
|
|
223
|
+
})[0];
|
|
224
|
+
const version =
|
|
225
|
+
production !== undefined && experiment.versions.includes(production) ? production : best;
|
|
226
|
+
if (version === undefined) {
|
|
227
|
+
continue;
|
|
228
|
+
}
|
|
229
|
+
const questionSet = questionSets.find((row) => {
|
|
230
|
+
return String(row._id) === String(experiment.questionSetId);
|
|
231
|
+
});
|
|
232
|
+
const gate = experiment.results?.gates.find((row) => row.version === version);
|
|
233
|
+
result.set(String(judge._id), {
|
|
234
|
+
datasetName: datasets.find((row) => String(row._id) === String(experiment.datasetId))?.name,
|
|
235
|
+
experimentId: String(experiment._id),
|
|
236
|
+
experimentName: experiment.name,
|
|
237
|
+
passed: gate?.passed ?? false,
|
|
238
|
+
questionSetName: questionSet?.name,
|
|
239
|
+
rate: agreement[String(version)]?.rate,
|
|
240
|
+
stale:
|
|
241
|
+
!questionSet ||
|
|
242
|
+
questionSetFingerprint(questionSet.dimensions) !== experiment.questionsFingerprint,
|
|
243
|
+
targetAgreement: experiment.targetAgreement,
|
|
244
|
+
version,
|
|
245
|
+
});
|
|
246
|
+
}
|
|
247
|
+
return result;
|
|
154
248
|
}
|
|
155
249
|
|
|
156
250
|
async getByName(name: string): Promise<EvaluatorView | undefined> {
|