@terreno/ai 58.0.0-beta.3 → 58.0.0-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. package/CHANGELOG.md +99 -0
  2. package/README.md +3 -3
  3. package/dist/harness/commit.d.ts +4 -3
  4. package/dist/harness/commit.js +23 -4
  5. package/dist/index.d.ts +6 -9
  6. package/dist/index.js +14 -25
  7. package/dist/observability/adminScreens.js +5 -1
  8. package/dist/observability/evaluate.d.ts +8 -1
  9. package/dist/observability/evaluate.js +10 -2
  10. package/dist/observability/judgeAgreement.d.ts +53 -0
  11. package/dist/observability/judgeAgreement.js +123 -0
  12. package/dist/observability/local/datasetStore.d.ts +5 -0
  13. package/dist/observability/local/datasetStore.js +7 -2
  14. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  15. package/dist/observability/local/evaluatorStore.js +73 -2
  16. package/dist/observability/local/experimentRunner.d.ts +40 -3
  17. package/dist/observability/local/experimentRunner.js +390 -11
  18. package/dist/observability/local/humanAnswers.d.ts +57 -0
  19. package/dist/observability/local/humanAnswers.js +174 -0
  20. package/dist/observability/local/judgeRun.d.ts +56 -0
  21. package/dist/observability/local/judgeRun.js +111 -0
  22. package/dist/observability/local/localPlugin.js +4 -0
  23. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  24. package/dist/observability/local/models/obsEvaluator.js +4 -0
  25. package/dist/observability/local/models/obsExperiment.js +28 -0
  26. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  27. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  28. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  29. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  30. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  31. package/dist/observability/local/models/obsReviewItem.js +10 -2
  32. package/dist/observability/local/models/obsTrace.js +4 -0
  33. package/dist/observability/local/promptReferences.d.ts +31 -0
  34. package/dist/observability/local/promptReferences.js +114 -0
  35. package/dist/observability/local/promptStore.d.ts +55 -19
  36. package/dist/observability/local/promptStore.js +171 -7
  37. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  38. package/dist/observability/local/reviewBatchStore.js +517 -0
  39. package/dist/observability/local/reviewStore.d.ts +24 -3
  40. package/dist/observability/local/reviewStore.js +98 -22
  41. package/dist/observability/local/traceStore.d.ts +14 -1
  42. package/dist/observability/local/traceStore.js +144 -11
  43. package/dist/observability/observabilityApp.js +10 -1
  44. package/dist/observability/questionFingerprint.d.ts +7 -0
  45. package/dist/observability/questionFingerprint.js +23 -0
  46. package/dist/observability/routes/experiments.js +12 -1
  47. package/dist/observability/routes/review.js +3 -1
  48. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  49. package/dist/observability/routes/reviewBatches.js +78 -0
  50. package/dist/observability/routes/traces.d.ts +1 -1
  51. package/dist/observability/routes/traces.js +34 -2
  52. package/dist/observability/types.d.ts +10 -1
  53. package/dist/service/aiService.js +4 -1
  54. package/dist/service/chatTurn.js +38 -31
  55. package/dist/types/harness.d.ts +5 -2
  56. package/dist/types/index.d.ts +8 -3
  57. package/dist/types/observability.d.ts +74 -2
  58. package/package.json +8 -24
  59. package/src/harness/agentLoop.test.ts +0 -3
  60. package/src/harness/agentRecovery.test.ts +0 -3
  61. package/src/harness/commit.ts +24 -4
  62. package/src/harness/extensions.test.ts +0 -3
  63. package/src/harness/subagent.test.ts +27 -3
  64. package/src/index.test.ts +0 -17
  65. package/src/index.ts +23 -26
  66. package/src/isolated/gptStream.isolated.ts +0 -8
  67. package/src/observability/adminScreens.test.ts +7 -0
  68. package/src/observability/adminScreens.ts +6 -1
  69. package/src/observability/evaluate.ts +12 -2
  70. package/src/observability/judgeAgreement.test.ts +111 -0
  71. package/src/observability/judgeAgreement.ts +184 -0
  72. package/src/observability/local/datasetStore.ts +30 -11
  73. package/src/observability/local/evaluatorStore.ts +96 -2
  74. package/src/observability/local/experimentRunner.ts +495 -15
  75. package/src/observability/local/humanAnswers.ts +233 -0
  76. package/src/observability/local/judgePipeline.test.ts +444 -0
  77. package/src/observability/local/judgeRun.test.ts +173 -0
  78. package/src/observability/local/judgeRun.ts +184 -0
  79. package/src/observability/local/localPlugin.ts +4 -0
  80. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  81. package/src/observability/local/models/obsEvaluator.ts +4 -0
  82. package/src/observability/local/models/obsExperiment.ts +29 -0
  83. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  84. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  85. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  86. package/src/observability/local/models/obsReviewItem.ts +11 -2
  87. package/src/observability/local/models/obsTrace.ts +5 -0
  88. package/src/observability/local/obsModels.test.ts +13 -0
  89. package/src/observability/local/promptReferences.test.ts +128 -0
  90. package/src/observability/local/promptReferences.ts +149 -0
  91. package/src/observability/local/promptStore.test.ts +291 -0
  92. package/src/observability/local/promptStore.ts +243 -32
  93. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  94. package/src/observability/local/reviewBatchStore.ts +656 -0
  95. package/src/observability/local/reviewStore.ts +122 -25
  96. package/src/observability/local/traceStore.test.ts +139 -1
  97. package/src/observability/local/traceStore.ts +162 -11
  98. package/src/observability/observabilityApp.test.ts +13 -10
  99. package/src/observability/observabilityApp.ts +10 -1
  100. package/src/observability/questionFingerprint.test.ts +33 -0
  101. package/src/observability/questionFingerprint.ts +24 -0
  102. package/src/observability/routes/experiments.test.ts +37 -0
  103. package/src/observability/routes/experiments.ts +13 -1
  104. package/src/observability/routes/review.ts +3 -1
  105. package/src/observability/routes/reviewBatches.test.ts +209 -0
  106. package/src/observability/routes/reviewBatches.ts +138 -0
  107. package/src/observability/routes/status.test.ts +4 -4
  108. package/src/observability/routes/traces.test.ts +61 -0
  109. package/src/observability/routes/traces.ts +46 -4
  110. package/src/observability/types.ts +5 -2
  111. package/src/routes/routes.test.ts +101 -76
  112. package/src/service/aiService.test.ts +21 -0
  113. package/src/service/aiService.ts +4 -1
  114. package/src/service/chatTurn.ts +42 -33
  115. package/src/tests/bunSetup.ts +0 -54
  116. package/src/types/harness.ts +5 -2
  117. package/src/types/index.ts +8 -3
  118. package/src/types/observability.ts +86 -2
  119. package/dist/langfuseApp.d.ts +0 -9
  120. package/dist/langfuseApp.js +0 -53
  121. package/dist/langfuseCache.d.ts +0 -15
  122. package/dist/langfuseCache.js +0 -54
  123. package/dist/langfuseClient.d.ts +0 -6
  124. package/dist/langfuseClient.js +0 -34
  125. package/dist/langfusePrompts.d.ts +0 -12
  126. package/dist/langfusePrompts.js +0 -89
  127. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  128. package/dist/langfuseRoutesEvaluations.js +0 -32
  129. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  130. package/dist/langfuseRoutesMiddleware.js +0 -13
  131. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  132. package/dist/langfuseRoutesPlayground.js +0 -36
  133. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  134. package/dist/langfuseRoutesPrompts.js +0 -56
  135. package/dist/langfuseRoutesTraces.d.ts +0 -2
  136. package/dist/langfuseRoutesTraces.js +0 -30
  137. package/dist/langfuseTracing.d.ts +0 -4
  138. package/dist/langfuseTracing.js +0 -32
  139. package/dist/langfuseTypes.d.ts +0 -96
  140. package/dist/langfuseTypes.js +0 -2
  141. package/dist/langfuseVercelAi.d.ts +0 -12
  142. package/dist/langfuseVercelAi.js +0 -49
  143. package/src/isolated/langfuseClient.isolated.ts +0 -66
  144. package/src/langfuseApp.test.ts +0 -119
  145. package/src/langfuseApp.ts +0 -62
  146. package/src/langfuseCache.test.ts +0 -75
  147. package/src/langfuseCache.ts +0 -72
  148. package/src/langfuseClient.test.ts +0 -69
  149. package/src/langfuseClient.ts +0 -34
  150. package/src/langfusePrompts.test.ts +0 -188
  151. package/src/langfusePrompts.ts +0 -121
  152. package/src/langfuseRoutes.test.ts +0 -261
  153. package/src/langfuseRoutesEvaluations.ts +0 -48
  154. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  155. package/src/langfuseRoutesMiddleware.ts +0 -12
  156. package/src/langfuseRoutesPlayground.ts +0 -50
  157. package/src/langfuseRoutesPrompts.ts +0 -87
  158. package/src/langfuseRoutesTraces.ts +0 -40
  159. package/src/langfuseTracing.test.ts +0 -72
  160. package/src/langfuseTracing.ts +0 -33
  161. package/src/langfuseTypes.ts +0 -103
  162. package/src/langfuseVercelAi.test.ts +0 -82
  163. package/src/langfuseVercelAi.ts +0 -72
  164. package/src/tests/fakeLangfusePrompt.ts +0 -10
@@ -9,6 +9,7 @@ const luxon_1 = require("luxon");
9
9
  const mongoose_1 = __importDefault(require("mongoose"));
10
10
  const datasetImport_1 = require("../datasetImport");
11
11
  const schemaValidation_1 = require("../schemaValidation");
12
+ const humanAnswers_1 = require("./humanAnswers");
12
13
  const obsDataset_1 = require("./models/obsDataset");
13
14
  const obsDatasetItem_1 = require("./models/obsDatasetItem");
14
15
  const obsTrace_1 = require("./models/obsTrace");
@@ -24,8 +25,9 @@ const throwOnDatasetImportErrors = (result) => {
24
25
  });
25
26
  };
26
27
  exports.throwOnDatasetImportErrors = throwOnDatasetImportErrors;
27
- const toDatasetView = async (doc) => {
28
+ const toDatasetView = async (doc, judgments) => {
28
29
  const counts = await computeDatasetCounts(String(doc._id));
30
+ const resolvedJudgments = judgments ?? (await (0, humanAnswers_1.summarizeDatasetJudgments)([doc._id])).get(String(doc._id)) ?? [];
29
31
  return {
30
32
  counts,
31
33
  created: luxon_1.DateTime.fromJSDate(doc.created).toUTC().toISO() ?? "",
@@ -33,6 +35,7 @@ const toDatasetView = async (doc) => {
33
35
  expectedOutputSchema: doc.expectedOutputSchema,
34
36
  id: String(doc._id),
35
37
  inputSchemaPromptName: doc.inputSchemaPromptName,
38
+ judgments: resolvedJudgments,
36
39
  name: doc.name,
37
40
  tags: doc.tags,
38
41
  updated: luxon_1.DateTime.fromJSDate(doc.updated).toUTC().toISO() ?? "",
@@ -44,6 +47,7 @@ const toItemView = (doc) => {
44
47
  created: luxon_1.DateTime.fromJSDate(doc.created).toUTC().toISO() ?? "",
45
48
  datasetId: String(doc.datasetId),
46
49
  expectedOutput: doc.expectedOutput,
50
+ humanAnswers: (0, humanAnswers_1.toHumanAnswerViews)(doc.humanAnswers),
47
51
  id: String(doc._id),
48
52
  input: doc.input,
49
53
  metadata: doc.metadata,
@@ -106,8 +110,9 @@ class LocalDatasetStore {
106
110
  }
107
111
  async list() {
108
112
  const rows = await (0, obsDataset_1.registerObsDataset)().find({}).sort({ name: 1 });
113
+ const judgments = await (0, humanAnswers_1.summarizeDatasetJudgments)(rows.map((row) => row._id));
109
114
  return Promise.all(rows.map((row) => {
110
- return toDatasetView(row);
115
+ return toDatasetView(row, judgments.get(String(row._id)) ?? []);
111
116
  }));
112
117
  }
113
118
  async get(id) {
@@ -15,11 +15,27 @@ export interface EvaluatorWriteInput {
15
15
  target: "dataset item" | "full trace" | "generation span";
16
16
  type: "human" | "json-assert" | "llm-judge";
17
17
  }
18
+ /** How well an llm-judge agreed with human answers in its latest judge calibration experiment. */
19
+ export interface EvaluatorCalibration {
20
+ datasetName?: string;
21
+ experimentId: string;
22
+ experimentName: string;
23
+ passed: boolean;
24
+ questionSetName?: string;
25
+ /** Agreement rate (0–1) of `version`; unset when nothing could be compared. */
26
+ rate?: number;
27
+ /** True when the question set's questions changed since the experiment ran. */
28
+ stale: boolean;
29
+ targetAgreement?: number;
30
+ /** The judge prompt's production version when it was compared, else the best version. */
31
+ version: number;
32
+ }
18
33
  export interface EvaluatorView {
19
34
  assertion?: {
20
35
  constraint: string;
21
36
  path: string;
22
37
  };
38
+ calibration?: EvaluatorCalibration;
23
39
  confidenceAlertBelow: number;
24
40
  description?: string;
25
41
  dimensions: EvaluatorDimension[];
@@ -37,6 +53,7 @@ export declare class LocalEvaluatorStore {
37
53
  create(input: EvaluatorWriteInput): Promise<EvaluatorView>;
38
54
  list(): Promise<EvaluatorView[]>;
39
55
  get(id: string): Promise<EvaluatorView>;
56
+ private loadCalibrations;
40
57
  getByName(name: string): Promise<EvaluatorView | undefined>;
41
58
  update(id: string, input: Partial<EvaluatorWriteInput>): Promise<EvaluatorView>;
42
59
  remove(id: string): Promise<void>;
@@ -4,7 +4,10 @@ exports.LocalEvaluatorStore = void 0;
4
4
  const api_1 = require("@terreno/api");
5
5
  const evaluate_1 = require("../evaluate");
6
6
  const evaluatorTemplates_1 = require("../evaluatorTemplates");
7
+ const questionFingerprint_1 = require("../questionFingerprint");
8
+ const obsDataset_1 = require("./models/obsDataset");
7
9
  const obsEvaluator_1 = require("./models/obsEvaluator");
10
+ const obsExperiment_1 = require("./models/obsExperiment");
8
11
  const promptStore_1 = require("./promptStore");
9
12
  const defaultRunModes = (partial) => {
10
13
  return {
@@ -102,8 +105,9 @@ class LocalEvaluatorStore {
102
105
  }
103
106
  async list() {
104
107
  const rows = await (0, obsEvaluator_1.registerObsEvaluator)().find({}).sort({ name: 1 });
108
+ const calibrations = await this.loadCalibrations(rows);
105
109
  return rows.map((row) => {
106
- return toView(row);
110
+ return { ...toView(row), calibration: calibrations.get(String(row._id)) };
107
111
  });
108
112
  }
109
113
  async get(id) {
@@ -111,7 +115,74 @@ class LocalEvaluatorStore {
111
115
  if (!doc) {
112
116
  throw new api_1.APIError({ status: 404, title: `Unknown evaluator "${id}"` });
113
117
  }
114
- return toView(doc);
118
+ const calibrations = await this.loadCalibrations([doc]);
119
+ return { ...toView(doc), calibration: calibrations.get(String(doc._id)) };
120
+ }
121
+ // Latest completed judge experiment per llm-judge evaluator, summarized for display.
122
+ async loadCalibrations(evaluators) {
123
+ const result = new Map();
124
+ const judges = evaluators.filter((row) => row.type === "llm-judge");
125
+ if (judges.length === 0) {
126
+ return result;
127
+ }
128
+ const experiments = await (0, obsExperiment_1.registerObsExperiment)()
129
+ .find({
130
+ judgeEvaluatorId: { $in: judges.map((row) => row._id) },
131
+ kind: "judge",
132
+ status: "completed",
133
+ })
134
+ .sort({ created: -1 });
135
+ const latest = new Map();
136
+ for (const experiment of experiments) {
137
+ const key = String(experiment.judgeEvaluatorId);
138
+ if (!latest.has(key)) {
139
+ latest.set(key, experiment);
140
+ }
141
+ }
142
+ if (latest.size === 0) {
143
+ return result;
144
+ }
145
+ const rows = [...latest.values()];
146
+ const datasets = await (0, obsDataset_1.registerObsDataset)().find({
147
+ _id: { $in: rows.map((row) => row.datasetId) },
148
+ });
149
+ const questionSets = await (0, obsEvaluator_1.registerObsEvaluator)().find({
150
+ _id: { $in: rows.flatMap((row) => (row.questionSetId ? [row.questionSetId] : [])) },
151
+ });
152
+ for (const judge of judges) {
153
+ const experiment = latest.get(String(judge._id));
154
+ if (!experiment) {
155
+ continue;
156
+ }
157
+ const production = judge.judgePromptName
158
+ ? (await this.promptStore.getVersionByLabel(judge.judgePromptName, "production"))?.version
159
+ : undefined;
160
+ const agreement = experiment.results?.agreement ?? {};
161
+ const best = [...experiment.versions].sort((left, right) => {
162
+ return (agreement[String(right)]?.rate ?? -1) - (agreement[String(left)]?.rate ?? -1);
163
+ })[0];
164
+ const version = production !== undefined && experiment.versions.includes(production) ? production : best;
165
+ if (version === undefined) {
166
+ continue;
167
+ }
168
+ const questionSet = questionSets.find((row) => {
169
+ return String(row._id) === String(experiment.questionSetId);
170
+ });
171
+ const gate = experiment.results?.gates.find((row) => row.version === version);
172
+ result.set(String(judge._id), {
173
+ datasetName: datasets.find((row) => String(row._id) === String(experiment.datasetId))?.name,
174
+ experimentId: String(experiment._id),
175
+ experimentName: experiment.name,
176
+ passed: gate?.passed ?? false,
177
+ questionSetName: questionSet?.name,
178
+ rate: agreement[String(version)]?.rate,
179
+ stale: !questionSet ||
180
+ (0, questionFingerprint_1.questionSetFingerprint)(questionSet.dimensions) !== experiment.questionsFingerprint,
181
+ targetAgreement: experiment.targetAgreement,
182
+ version,
183
+ });
184
+ }
185
+ return result;
115
186
  }
116
187
  async getByName(name) {
117
188
  const doc = await (0, obsEvaluator_1.registerObsEvaluator)().findOneOrNone({ name });
@@ -1,4 +1,4 @@
1
- import type { ExperimentAggregates, ExperimentEstimate, ExperimentVersionResult, ScoreThreshold } from "../../types/observability";
1
+ import type { ExperimentAggregates, ExperimentEstimate, ExperimentKind, ExperimentVersionResult, ScoreThreshold } from "../../types/observability";
2
2
  import type { ModelPrice, ObservabilityAiServiceFactory, ObservabilityGenerateClient } from "../types";
3
3
  import type { LocalDatasetStore } from "./datasetStore";
4
4
  import type { LocalEvaluatorStore } from "./evaluatorStore";
@@ -7,26 +7,54 @@ export interface ExperimentCreateInput {
7
7
  datasetId: string;
8
8
  evaluatorIds: string[];
9
9
  includeUnproofread?: boolean;
10
+ kind?: "prompt";
10
11
  modelOverride?: string;
11
12
  name: string;
12
13
  promptName: string;
13
14
  thresholds?: ScoreThreshold[];
14
15
  versions: number[];
15
16
  }
17
+ /**
18
+ * Judge calibration: run 2–3 versions of an llm-judge's prompt over dataset items humans already
19
+ * answered with `questionSetId`, and rank versions by agreement with those answers.
20
+ */
21
+ export interface JudgeExperimentCreateInput {
22
+ datasetId: string;
23
+ judgeEvaluatorId: string;
24
+ kind: "judge";
25
+ modelOverride?: string;
26
+ name: string;
27
+ questionSetId: string;
28
+ /** Agreement rate (0–1) a version must reach to pass its gate. Defaults to 0.9. */
29
+ targetAgreement?: number;
30
+ /** Versions of the judge evaluator's `judgePromptName`. */
31
+ versions: number[];
32
+ }
33
+ export declare const DEFAULT_TARGET_AGREEMENT = 0.9;
16
34
  export interface ExperimentView {
17
35
  backgroundTaskId?: string;
18
36
  created: string;
19
37
  datasetId: string;
38
+ datasetName?: string;
20
39
  estimate?: ExperimentEstimate;
21
40
  evaluatorIds: string[];
22
41
  id: string;
23
42
  includeUnproofread: boolean;
24
43
  items: ExperimentItemView[];
44
+ judgeEvaluatorId?: string;
45
+ judgeName?: string;
46
+ kind: ExperimentKind;
25
47
  modelOverride?: string;
26
48
  name: string;
27
49
  promptName: string;
50
+ questionSetId?: string;
51
+ questionSetName?: string;
52
+ questionsFingerprint?: string;
53
+ /** Judge experiments: true when the question set's questions changed since the run. */
54
+ questionsStale?: boolean;
28
55
  results?: ExperimentAggregates;
29
56
  status: "completed" | "failed" | "pending" | "running";
57
+ targetAgreement?: number;
30
58
  thresholds: ScoreThreshold[];
31
59
  updated: string;
32
60
  versions: number[];
@@ -34,6 +62,8 @@ export interface ExperimentView {
34
62
  export interface ExperimentItemView {
35
63
  datasetItemId: string;
36
64
  failed: boolean;
65
+ /** Judge experiments: the human answers each judge version is compared with. */
66
+ humanScores?: Record<string, boolean | number | string>;
37
67
  id: string;
38
68
  versionResults: Record<string, ExperimentVersionResult>;
39
69
  }
@@ -73,21 +103,28 @@ export declare class LocalExperimentRunner {
73
103
  private resolveAiClient;
74
104
  estimate(input: {
75
105
  datasetId: string;
76
- evaluatorIds: string[];
106
+ evaluatorIds?: string[];
77
107
  includeUnproofread?: boolean;
108
+ kind?: ExperimentKind;
78
109
  modelOverride?: string;
110
+ questionSetId?: string;
79
111
  versions: number[];
80
112
  }): Promise<ExperimentEstimate>;
81
113
  private assertAutomaticEvaluators;
82
- create(input: ExperimentCreateInput): Promise<ExperimentView>;
114
+ create(input: ExperimentCreateInput | JudgeExperimentCreateInput): Promise<ExperimentView>;
115
+ private createJudge;
116
+ private startRun;
83
117
  list(params?: {
118
+ kind?: ExperimentKind;
84
119
  limit?: number;
85
120
  page?: number;
86
121
  promptName?: string;
87
122
  }): Promise<ExperimentListResult>;
88
123
  get(id: string): Promise<ExperimentView>;
124
+ private loadNames;
89
125
  private toSummaryView;
90
126
  promote(id: string, version: number): Promise<MoveLabelResult>;
91
127
  executeExperiment(id: string): Promise<void>;
128
+ private executeJudgeExperiment;
92
129
  private computeGates;
93
130
  }