@terreno/ai 57.21.0 → 57.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/README.md +3 -3
- package/biome.jsonc +2 -2
- package/dist/harness/commit.d.ts +4 -3
- package/dist/harness/commit.js +23 -4
- package/dist/index.d.ts +6 -9
- package/dist/index.js +14 -25
- package/dist/observability/adminScreens.js +5 -1
- package/dist/observability/evaluate.d.ts +8 -1
- package/dist/observability/evaluate.js +10 -2
- package/dist/observability/judgeAgreement.d.ts +53 -0
- package/dist/observability/judgeAgreement.js +123 -0
- package/dist/observability/local/datasetStore.d.ts +5 -0
- package/dist/observability/local/datasetStore.js +7 -2
- package/dist/observability/local/evaluatorStore.d.ts +17 -0
- package/dist/observability/local/evaluatorStore.js +73 -2
- package/dist/observability/local/experimentRunner.d.ts +40 -3
- package/dist/observability/local/experimentRunner.js +390 -11
- package/dist/observability/local/humanAnswers.d.ts +57 -0
- package/dist/observability/local/humanAnswers.js +174 -0
- package/dist/observability/local/judgeRun.d.ts +56 -0
- package/dist/observability/local/judgeRun.js +111 -0
- package/dist/observability/local/localPlugin.js +4 -0
- package/dist/observability/local/models/obsDatasetItem.js +4 -0
- package/dist/observability/local/models/obsEvaluator.js +4 -0
- package/dist/observability/local/models/obsExperiment.js +28 -0
- package/dist/observability/local/models/obsExperimentItem.js +4 -0
- package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
- package/dist/observability/local/models/obsJudgeRun.js +60 -0
- package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
- package/dist/observability/local/models/obsReviewBatch.js +48 -0
- package/dist/observability/local/models/obsReviewItem.js +10 -2
- package/dist/observability/local/models/obsTrace.js +4 -0
- package/dist/observability/local/promptReferences.d.ts +31 -0
- package/dist/observability/local/promptReferences.js +114 -0
- package/dist/observability/local/promptStore.d.ts +55 -19
- package/dist/observability/local/promptStore.js +171 -7
- package/dist/observability/local/reviewBatchStore.d.ts +106 -0
- package/dist/observability/local/reviewBatchStore.js +517 -0
- package/dist/observability/local/reviewStore.d.ts +24 -3
- package/dist/observability/local/reviewStore.js +98 -22
- package/dist/observability/local/traceStore.d.ts +14 -1
- package/dist/observability/local/traceStore.js +144 -11
- package/dist/observability/observabilityApp.js +10 -1
- package/dist/observability/questionFingerprint.d.ts +7 -0
- package/dist/observability/questionFingerprint.js +23 -0
- package/dist/observability/routes/experiments.js +12 -1
- package/dist/observability/routes/review.js +3 -1
- package/dist/observability/routes/reviewBatches.d.ts +11 -0
- package/dist/observability/routes/reviewBatches.js +78 -0
- package/dist/observability/routes/traces.d.ts +1 -1
- package/dist/observability/routes/traces.js +34 -2
- package/dist/observability/types.d.ts +10 -1
- package/dist/service/aiService.js +4 -1
- package/dist/service/chatTurn.js +38 -31
- package/dist/types/harness.d.ts +5 -2
- package/dist/types/index.d.ts +8 -3
- package/dist/types/observability.d.ts +74 -2
- package/package.json +8 -24
- package/src/harness/agentLoop.test.ts +0 -3
- package/src/harness/agentRecovery.test.ts +0 -3
- package/src/harness/commit.ts +24 -4
- package/src/harness/extensions.test.ts +0 -3
- package/src/harness/subagent.test.ts +27 -3
- package/src/index.test.ts +0 -17
- package/src/index.ts +23 -26
- package/src/isolated/gptStream.isolated.ts +0 -8
- package/src/observability/adminScreens.test.ts +7 -0
- package/src/observability/adminScreens.ts +6 -1
- package/src/observability/evaluate.ts +12 -2
- package/src/observability/judgeAgreement.test.ts +111 -0
- package/src/observability/judgeAgreement.ts +184 -0
- package/src/observability/local/datasetStore.ts +30 -11
- package/src/observability/local/evaluatorStore.ts +96 -2
- package/src/observability/local/experimentRunner.ts +495 -15
- package/src/observability/local/humanAnswers.ts +233 -0
- package/src/observability/local/judgePipeline.test.ts +444 -0
- package/src/observability/local/judgeRun.test.ts +173 -0
- package/src/observability/local/judgeRun.ts +184 -0
- package/src/observability/local/localPlugin.ts +4 -0
- package/src/observability/local/models/obsDatasetItem.ts +5 -0
- package/src/observability/local/models/obsEvaluator.ts +4 -0
- package/src/observability/local/models/obsExperiment.ts +29 -0
- package/src/observability/local/models/obsExperimentItem.ts +4 -0
- package/src/observability/local/models/obsJudgeRun.ts +61 -0
- package/src/observability/local/models/obsReviewBatch.ts +52 -0
- package/src/observability/local/models/obsReviewItem.ts +11 -2
- package/src/observability/local/models/obsTrace.ts +5 -0
- package/src/observability/local/obsModels.test.ts +13 -0
- package/src/observability/local/promptReferences.test.ts +128 -0
- package/src/observability/local/promptReferences.ts +149 -0
- package/src/observability/local/promptStore.test.ts +291 -0
- package/src/observability/local/promptStore.ts +243 -32
- package/src/observability/local/reviewBatchStore.test.ts +251 -0
- package/src/observability/local/reviewBatchStore.ts +656 -0
- package/src/observability/local/reviewStore.ts +122 -25
- package/src/observability/local/traceStore.test.ts +139 -1
- package/src/observability/local/traceStore.ts +162 -11
- package/src/observability/observabilityApp.test.ts +13 -10
- package/src/observability/observabilityApp.ts +10 -1
- package/src/observability/questionFingerprint.test.ts +33 -0
- package/src/observability/questionFingerprint.ts +24 -0
- package/src/observability/routes/experiments.test.ts +37 -0
- package/src/observability/routes/experiments.ts +13 -1
- package/src/observability/routes/review.ts +3 -1
- package/src/observability/routes/reviewBatches.test.ts +209 -0
- package/src/observability/routes/reviewBatches.ts +138 -0
- package/src/observability/routes/status.test.ts +4 -4
- package/src/observability/routes/traces.test.ts +61 -0
- package/src/observability/routes/traces.ts +46 -4
- package/src/observability/types.ts +5 -2
- package/src/routes/routes.test.ts +101 -76
- package/src/service/aiService.test.ts +21 -0
- package/src/service/aiService.ts +4 -1
- package/src/service/chatTurn.ts +42 -33
- package/src/tests/bunSetup.ts +0 -54
- package/src/types/harness.ts +5 -2
- package/src/types/index.ts +8 -3
- package/src/types/observability.ts +86 -2
- package/dist/langfuseApp.d.ts +0 -9
- package/dist/langfuseApp.js +0 -53
- package/dist/langfuseCache.d.ts +0 -15
- package/dist/langfuseCache.js +0 -54
- package/dist/langfuseClient.d.ts +0 -6
- package/dist/langfuseClient.js +0 -34
- package/dist/langfusePrompts.d.ts +0 -12
- package/dist/langfusePrompts.js +0 -89
- package/dist/langfuseRoutesEvaluations.d.ts +0 -3
- package/dist/langfuseRoutesEvaluations.js +0 -32
- package/dist/langfuseRoutesMiddleware.d.ts +0 -2
- package/dist/langfuseRoutesMiddleware.js +0 -13
- package/dist/langfuseRoutesPlayground.d.ts +0 -2
- package/dist/langfuseRoutesPlayground.js +0 -36
- package/dist/langfuseRoutesPrompts.d.ts +0 -2
- package/dist/langfuseRoutesPrompts.js +0 -56
- package/dist/langfuseRoutesTraces.d.ts +0 -2
- package/dist/langfuseRoutesTraces.js +0 -30
- package/dist/langfuseTracing.d.ts +0 -4
- package/dist/langfuseTracing.js +0 -32
- package/dist/langfuseTypes.d.ts +0 -96
- package/dist/langfuseTypes.js +0 -2
- package/dist/langfuseVercelAi.d.ts +0 -12
- package/dist/langfuseVercelAi.js +0 -49
- package/src/isolated/langfuseClient.isolated.ts +0 -66
- package/src/langfuseApp.test.ts +0 -119
- package/src/langfuseApp.ts +0 -62
- package/src/langfuseCache.test.ts +0 -75
- package/src/langfuseCache.ts +0 -72
- package/src/langfuseClient.test.ts +0 -69
- package/src/langfuseClient.ts +0 -34
- package/src/langfusePrompts.test.ts +0 -188
- package/src/langfusePrompts.ts +0 -121
- package/src/langfuseRoutes.test.ts +0 -261
- package/src/langfuseRoutesEvaluations.ts +0 -48
- package/src/langfuseRoutesMiddleware.test.ts +0 -57
- package/src/langfuseRoutesMiddleware.ts +0 -12
- package/src/langfuseRoutesPlayground.ts +0 -50
- package/src/langfuseRoutesPrompts.ts +0 -87
- package/src/langfuseRoutesTraces.ts +0 -40
- package/src/langfuseTracing.test.ts +0 -72
- package/src/langfuseTracing.ts +0 -33
- package/src/langfuseTypes.ts +0 -103
- package/src/langfuseVercelAi.test.ts +0 -82
- package/src/langfuseVercelAi.ts +0 -72
- package/src/tests/fakeLangfusePrompt.ts +0 -10
|
@@ -4,7 +4,10 @@ exports.LocalEvaluatorStore = void 0;
|
|
|
4
4
|
const api_1 = require("@terreno/api");
|
|
5
5
|
const evaluate_1 = require("../evaluate");
|
|
6
6
|
const evaluatorTemplates_1 = require("../evaluatorTemplates");
|
|
7
|
+
const questionFingerprint_1 = require("../questionFingerprint");
|
|
8
|
+
const obsDataset_1 = require("./models/obsDataset");
|
|
7
9
|
const obsEvaluator_1 = require("./models/obsEvaluator");
|
|
10
|
+
const obsExperiment_1 = require("./models/obsExperiment");
|
|
8
11
|
const promptStore_1 = require("./promptStore");
|
|
9
12
|
const defaultRunModes = (partial) => {
|
|
10
13
|
return {
|
|
@@ -102,8 +105,9 @@ class LocalEvaluatorStore {
|
|
|
102
105
|
}
|
|
103
106
|
async list() {
|
|
104
107
|
const rows = await (0, obsEvaluator_1.registerObsEvaluator)().find({}).sort({ name: 1 });
|
|
108
|
+
const calibrations = await this.loadCalibrations(rows);
|
|
105
109
|
return rows.map((row) => {
|
|
106
|
-
return toView(row);
|
|
110
|
+
return { ...toView(row), calibration: calibrations.get(String(row._id)) };
|
|
107
111
|
});
|
|
108
112
|
}
|
|
109
113
|
async get(id) {
|
|
@@ -111,7 +115,74 @@ class LocalEvaluatorStore {
|
|
|
111
115
|
if (!doc) {
|
|
112
116
|
throw new api_1.APIError({ status: 404, title: `Unknown evaluator "${id}"` });
|
|
113
117
|
}
|
|
114
|
-
|
|
118
|
+
const calibrations = await this.loadCalibrations([doc]);
|
|
119
|
+
return { ...toView(doc), calibration: calibrations.get(String(doc._id)) };
|
|
120
|
+
}
|
|
121
|
+
// Latest completed judge experiment per llm-judge evaluator, summarized for display.
|
|
122
|
+
async loadCalibrations(evaluators) {
|
|
123
|
+
const result = new Map();
|
|
124
|
+
const judges = evaluators.filter((row) => row.type === "llm-judge");
|
|
125
|
+
if (judges.length === 0) {
|
|
126
|
+
return result;
|
|
127
|
+
}
|
|
128
|
+
const experiments = await (0, obsExperiment_1.registerObsExperiment)()
|
|
129
|
+
.find({
|
|
130
|
+
judgeEvaluatorId: { $in: judges.map((row) => row._id) },
|
|
131
|
+
kind: "judge",
|
|
132
|
+
status: "completed",
|
|
133
|
+
})
|
|
134
|
+
.sort({ created: -1 });
|
|
135
|
+
const latest = new Map();
|
|
136
|
+
for (const experiment of experiments) {
|
|
137
|
+
const key = String(experiment.judgeEvaluatorId);
|
|
138
|
+
if (!latest.has(key)) {
|
|
139
|
+
latest.set(key, experiment);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
if (latest.size === 0) {
|
|
143
|
+
return result;
|
|
144
|
+
}
|
|
145
|
+
const rows = [...latest.values()];
|
|
146
|
+
const datasets = await (0, obsDataset_1.registerObsDataset)().find({
|
|
147
|
+
_id: { $in: rows.map((row) => row.datasetId) },
|
|
148
|
+
});
|
|
149
|
+
const questionSets = await (0, obsEvaluator_1.registerObsEvaluator)().find({
|
|
150
|
+
_id: { $in: rows.flatMap((row) => (row.questionSetId ? [row.questionSetId] : [])) },
|
|
151
|
+
});
|
|
152
|
+
for (const judge of judges) {
|
|
153
|
+
const experiment = latest.get(String(judge._id));
|
|
154
|
+
if (!experiment) {
|
|
155
|
+
continue;
|
|
156
|
+
}
|
|
157
|
+
const production = judge.judgePromptName
|
|
158
|
+
? (await this.promptStore.getVersionByLabel(judge.judgePromptName, "production"))?.version
|
|
159
|
+
: undefined;
|
|
160
|
+
const agreement = experiment.results?.agreement ?? {};
|
|
161
|
+
const best = [...experiment.versions].sort((left, right) => {
|
|
162
|
+
return (agreement[String(right)]?.rate ?? -1) - (agreement[String(left)]?.rate ?? -1);
|
|
163
|
+
})[0];
|
|
164
|
+
const version = production !== undefined && experiment.versions.includes(production) ? production : best;
|
|
165
|
+
if (version === undefined) {
|
|
166
|
+
continue;
|
|
167
|
+
}
|
|
168
|
+
const questionSet = questionSets.find((row) => {
|
|
169
|
+
return String(row._id) === String(experiment.questionSetId);
|
|
170
|
+
});
|
|
171
|
+
const gate = experiment.results?.gates.find((row) => row.version === version);
|
|
172
|
+
result.set(String(judge._id), {
|
|
173
|
+
datasetName: datasets.find((row) => String(row._id) === String(experiment.datasetId))?.name,
|
|
174
|
+
experimentId: String(experiment._id),
|
|
175
|
+
experimentName: experiment.name,
|
|
176
|
+
passed: gate?.passed ?? false,
|
|
177
|
+
questionSetName: questionSet?.name,
|
|
178
|
+
rate: agreement[String(version)]?.rate,
|
|
179
|
+
stale: !questionSet ||
|
|
180
|
+
(0, questionFingerprint_1.questionSetFingerprint)(questionSet.dimensions) !== experiment.questionsFingerprint,
|
|
181
|
+
targetAgreement: experiment.targetAgreement,
|
|
182
|
+
version,
|
|
183
|
+
});
|
|
184
|
+
}
|
|
185
|
+
return result;
|
|
115
186
|
}
|
|
116
187
|
async getByName(name) {
|
|
117
188
|
const doc = await (0, obsEvaluator_1.registerObsEvaluator)().findOneOrNone({ name });
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ExperimentAggregates, ExperimentEstimate, ExperimentVersionResult, ScoreThreshold } from "../../types/observability";
|
|
1
|
+
import type { ExperimentAggregates, ExperimentEstimate, ExperimentKind, ExperimentVersionResult, ScoreThreshold } from "../../types/observability";
|
|
2
2
|
import type { ModelPrice, ObservabilityAiServiceFactory, ObservabilityGenerateClient } from "../types";
|
|
3
3
|
import type { LocalDatasetStore } from "./datasetStore";
|
|
4
4
|
import type { LocalEvaluatorStore } from "./evaluatorStore";
|
|
@@ -7,26 +7,54 @@ export interface ExperimentCreateInput {
|
|
|
7
7
|
datasetId: string;
|
|
8
8
|
evaluatorIds: string[];
|
|
9
9
|
includeUnproofread?: boolean;
|
|
10
|
+
kind?: "prompt";
|
|
10
11
|
modelOverride?: string;
|
|
11
12
|
name: string;
|
|
12
13
|
promptName: string;
|
|
13
14
|
thresholds?: ScoreThreshold[];
|
|
14
15
|
versions: number[];
|
|
15
16
|
}
|
|
17
|
+
/**
|
|
18
|
+
* Judge calibration: run 2–3 versions of an llm-judge's prompt over dataset items humans already
|
|
19
|
+
* answered with `questionSetId`, and rank versions by agreement with those answers.
|
|
20
|
+
*/
|
|
21
|
+
export interface JudgeExperimentCreateInput {
|
|
22
|
+
datasetId: string;
|
|
23
|
+
judgeEvaluatorId: string;
|
|
24
|
+
kind: "judge";
|
|
25
|
+
modelOverride?: string;
|
|
26
|
+
name: string;
|
|
27
|
+
questionSetId: string;
|
|
28
|
+
/** Agreement rate (0–1) a version must reach to pass its gate. Defaults to 0.9. */
|
|
29
|
+
targetAgreement?: number;
|
|
30
|
+
/** Versions of the judge evaluator's `judgePromptName`. */
|
|
31
|
+
versions: number[];
|
|
32
|
+
}
|
|
33
|
+
export declare const DEFAULT_TARGET_AGREEMENT = 0.9;
|
|
16
34
|
export interface ExperimentView {
|
|
17
35
|
backgroundTaskId?: string;
|
|
18
36
|
created: string;
|
|
19
37
|
datasetId: string;
|
|
38
|
+
datasetName?: string;
|
|
20
39
|
estimate?: ExperimentEstimate;
|
|
21
40
|
evaluatorIds: string[];
|
|
22
41
|
id: string;
|
|
23
42
|
includeUnproofread: boolean;
|
|
24
43
|
items: ExperimentItemView[];
|
|
44
|
+
judgeEvaluatorId?: string;
|
|
45
|
+
judgeName?: string;
|
|
46
|
+
kind: ExperimentKind;
|
|
25
47
|
modelOverride?: string;
|
|
26
48
|
name: string;
|
|
27
49
|
promptName: string;
|
|
50
|
+
questionSetId?: string;
|
|
51
|
+
questionSetName?: string;
|
|
52
|
+
questionsFingerprint?: string;
|
|
53
|
+
/** Judge experiments: true when the question set's questions changed since the run. */
|
|
54
|
+
questionsStale?: boolean;
|
|
28
55
|
results?: ExperimentAggregates;
|
|
29
56
|
status: "completed" | "failed" | "pending" | "running";
|
|
57
|
+
targetAgreement?: number;
|
|
30
58
|
thresholds: ScoreThreshold[];
|
|
31
59
|
updated: string;
|
|
32
60
|
versions: number[];
|
|
@@ -34,6 +62,8 @@ export interface ExperimentView {
|
|
|
34
62
|
export interface ExperimentItemView {
|
|
35
63
|
datasetItemId: string;
|
|
36
64
|
failed: boolean;
|
|
65
|
+
/** Judge experiments: the human answers each judge version is compared with. */
|
|
66
|
+
humanScores?: Record<string, boolean | number | string>;
|
|
37
67
|
id: string;
|
|
38
68
|
versionResults: Record<string, ExperimentVersionResult>;
|
|
39
69
|
}
|
|
@@ -73,21 +103,28 @@ export declare class LocalExperimentRunner {
|
|
|
73
103
|
private resolveAiClient;
|
|
74
104
|
estimate(input: {
|
|
75
105
|
datasetId: string;
|
|
76
|
-
evaluatorIds
|
|
106
|
+
evaluatorIds?: string[];
|
|
77
107
|
includeUnproofread?: boolean;
|
|
108
|
+
kind?: ExperimentKind;
|
|
78
109
|
modelOverride?: string;
|
|
110
|
+
questionSetId?: string;
|
|
79
111
|
versions: number[];
|
|
80
112
|
}): Promise<ExperimentEstimate>;
|
|
81
113
|
private assertAutomaticEvaluators;
|
|
82
|
-
create(input: ExperimentCreateInput): Promise<ExperimentView>;
|
|
114
|
+
create(input: ExperimentCreateInput | JudgeExperimentCreateInput): Promise<ExperimentView>;
|
|
115
|
+
private createJudge;
|
|
116
|
+
private startRun;
|
|
83
117
|
list(params?: {
|
|
118
|
+
kind?: ExperimentKind;
|
|
84
119
|
limit?: number;
|
|
85
120
|
page?: number;
|
|
86
121
|
promptName?: string;
|
|
87
122
|
}): Promise<ExperimentListResult>;
|
|
88
123
|
get(id: string): Promise<ExperimentView>;
|
|
124
|
+
private loadNames;
|
|
89
125
|
private toSummaryView;
|
|
90
126
|
promote(id: string, version: number): Promise<MoveLabelResult>;
|
|
91
127
|
executeExperiment(id: string): Promise<void>;
|
|
128
|
+
private executeJudgeExperiment;
|
|
92
129
|
private computeGates;
|
|
93
130
|
}
|