@terreno/ai 57.20.0 → 57.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. package/CHANGELOG.md +81 -0
  2. package/README.md +3 -3
  3. package/biome.jsonc +3 -2
  4. package/dist/harness/agentLoop.js +1 -0
  5. package/dist/harness/commit.d.ts +4 -3
  6. package/dist/harness/commit.js +23 -4
  7. package/dist/index.d.ts +6 -9
  8. package/dist/index.js +14 -25
  9. package/dist/observability/adminScreens.js +5 -1
  10. package/dist/observability/evaluate.d.ts +8 -1
  11. package/dist/observability/evaluate.js +10 -2
  12. package/dist/observability/judgeAgreement.d.ts +53 -0
  13. package/dist/observability/judgeAgreement.js +123 -0
  14. package/dist/observability/local/datasetStore.d.ts +5 -0
  15. package/dist/observability/local/datasetStore.js +7 -2
  16. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  17. package/dist/observability/local/evaluatorStore.js +73 -2
  18. package/dist/observability/local/experimentRunner.d.ts +42 -8
  19. package/dist/observability/local/experimentRunner.js +390 -11
  20. package/dist/observability/local/humanAnswers.d.ts +57 -0
  21. package/dist/observability/local/humanAnswers.js +174 -0
  22. package/dist/observability/local/judgeRun.d.ts +56 -0
  23. package/dist/observability/local/judgeRun.js +111 -0
  24. package/dist/observability/local/localPlugin.js +4 -0
  25. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  26. package/dist/observability/local/models/obsEvaluator.js +4 -0
  27. package/dist/observability/local/models/obsExperiment.js +28 -0
  28. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  29. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  30. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  31. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  32. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  33. package/dist/observability/local/models/obsReviewItem.js +10 -2
  34. package/dist/observability/local/models/obsTrace.js +4 -0
  35. package/dist/observability/local/promptReferences.d.ts +31 -0
  36. package/dist/observability/local/promptReferences.js +114 -0
  37. package/dist/observability/local/promptStore.d.ts +55 -19
  38. package/dist/observability/local/promptStore.js +171 -7
  39. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  40. package/dist/observability/local/reviewBatchStore.js +517 -0
  41. package/dist/observability/local/reviewStore.d.ts +24 -3
  42. package/dist/observability/local/reviewStore.js +98 -22
  43. package/dist/observability/local/traceStore.d.ts +14 -1
  44. package/dist/observability/local/traceStore.js +144 -11
  45. package/dist/observability/observabilityApp.js +10 -1
  46. package/dist/observability/questionFingerprint.d.ts +7 -0
  47. package/dist/observability/questionFingerprint.js +23 -0
  48. package/dist/observability/routes/experiments.js +12 -1
  49. package/dist/observability/routes/review.js +3 -1
  50. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  51. package/dist/observability/routes/reviewBatches.js +78 -0
  52. package/dist/observability/routes/traces.d.ts +1 -1
  53. package/dist/observability/routes/traces.js +34 -2
  54. package/dist/observability/types.d.ts +10 -1
  55. package/dist/service/aiService.js +4 -1
  56. package/dist/service/chatTurn.js +38 -31
  57. package/dist/types/harness.d.ts +10 -4
  58. package/dist/types/index.d.ts +8 -3
  59. package/dist/types/observability.d.ts +74 -2
  60. package/e2e/smoke.ts +227 -0
  61. package/package.json +14 -28
  62. package/src/aiApp.test.ts +22 -17
  63. package/src/harness/agentLoop.test.ts +0 -3
  64. package/src/harness/agentLoop.ts +2 -1
  65. package/src/harness/agentRecovery.test.ts +30 -30
  66. package/src/harness/approvals.test.ts +11 -9
  67. package/src/harness/commit.ts +24 -4
  68. package/src/harness/events.test.ts +10 -6
  69. package/src/harness/extensions.test.ts +6 -8
  70. package/src/harness/ownership.test.ts +20 -9
  71. package/src/harness/retries.test.ts +6 -3
  72. package/src/harness/routes/conversations.test.ts +2 -2
  73. package/src/harness/routes/tasks.test.ts +2 -2
  74. package/src/harness/runners/jobsRunner.test.ts +1 -1
  75. package/src/harness/subagent.test.ts +29 -4
  76. package/src/harness/waits.test.ts +6 -4
  77. package/src/index.test.ts +3 -20
  78. package/src/index.ts +23 -26
  79. package/src/isolated/gptStream.isolated.ts +25 -28
  80. package/src/isolated/lazyProjectModel.isolated.ts +2 -2
  81. package/src/models/gptHistory.test.ts +10 -6
  82. package/src/observability/adminScreens.test.ts +7 -0
  83. package/src/observability/adminScreens.ts +6 -1
  84. package/src/observability/evaluate.test.ts +12 -10
  85. package/src/observability/evaluate.ts +12 -2
  86. package/src/observability/judgeAgreement.test.ts +111 -0
  87. package/src/observability/judgeAgreement.ts +184 -0
  88. package/src/observability/local/datasetStore.ts +30 -11
  89. package/src/observability/local/evaluatorStore.ts +96 -2
  90. package/src/observability/local/experimentRunner.test.ts +9 -12
  91. package/src/observability/local/experimentRunner.ts +497 -17
  92. package/src/observability/local/humanAnswers.ts +233 -0
  93. package/src/observability/local/judgePipeline.test.ts +444 -0
  94. package/src/observability/local/judgeRun.test.ts +173 -0
  95. package/src/observability/local/judgeRun.ts +184 -0
  96. package/src/observability/local/localPlugin.ts +4 -0
  97. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  98. package/src/observability/local/models/obsEvaluator.ts +4 -0
  99. package/src/observability/local/models/obsExperiment.ts +29 -0
  100. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  101. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  102. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  103. package/src/observability/local/models/obsReviewItem.ts +11 -2
  104. package/src/observability/local/models/obsTrace.ts +5 -0
  105. package/src/observability/local/obsModels.test.ts +13 -0
  106. package/src/observability/local/promptReferences.test.ts +128 -0
  107. package/src/observability/local/promptReferences.ts +149 -0
  108. package/src/observability/local/promptStore.test.ts +295 -3
  109. package/src/observability/local/promptStore.ts +243 -32
  110. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  111. package/src/observability/local/reviewBatchStore.ts +656 -0
  112. package/src/observability/local/reviewStore.ts +122 -25
  113. package/src/observability/local/traceStore.test.ts +139 -1
  114. package/src/observability/local/traceStore.ts +162 -11
  115. package/src/observability/observabilityApp.test.ts +13 -10
  116. package/src/observability/observabilityApp.ts +10 -1
  117. package/src/observability/questionFingerprint.test.ts +33 -0
  118. package/src/observability/questionFingerprint.ts +24 -0
  119. package/src/observability/routes/datasets.test.ts +2 -2
  120. package/src/observability/routes/evaluators.test.ts +2 -2
  121. package/src/observability/routes/experiments.test.ts +39 -2
  122. package/src/observability/routes/experiments.ts +13 -1
  123. package/src/observability/routes/prompts.test.ts +3 -3
  124. package/src/observability/routes/review.test.ts +2 -2
  125. package/src/observability/routes/review.ts +3 -1
  126. package/src/observability/routes/reviewBatches.test.ts +209 -0
  127. package/src/observability/routes/reviewBatches.ts +138 -0
  128. package/src/observability/routes/status.test.ts +8 -8
  129. package/src/observability/routes/testMultiStage.test.ts +6 -6
  130. package/src/observability/routes/traces.test.ts +63 -2
  131. package/src/observability/routes/traces.ts +46 -4
  132. package/src/observability/types.ts +5 -2
  133. package/src/routes/files.test.ts +5 -3
  134. package/src/routes/gpt.test.ts +21 -14
  135. package/src/routes/gptActions.test.ts +7 -8
  136. package/src/routes/gptDatasets.test.ts +1 -1
  137. package/src/routes/gptFiles.test.ts +1 -1
  138. package/src/routes/gptHistories.test.ts +15 -9
  139. package/src/routes/mcp.test.ts +3 -3
  140. package/src/routes/projects.test.ts +5 -3
  141. package/src/routes/routes.test.ts +156 -115
  142. package/src/service/aiService.test.ts +22 -1
  143. package/src/service/aiService.ts +4 -1
  144. package/src/service/asks.test.ts +14 -6
  145. package/src/service/chatTurn.ts +42 -33
  146. package/src/service/fileStorage.test.ts +5 -2
  147. package/src/service/getMCPTools.test.ts +9 -5
  148. package/src/service/mcpService.test.ts +6 -3
  149. package/src/tests/bunSetup.ts +0 -54
  150. package/src/tests/chatHarness.ts +4 -4
  151. package/src/tests/fakeJudge.ts +8 -0
  152. package/src/tests/helpers.ts +26 -4
  153. package/src/types/harness.ts +11 -4
  154. package/src/types/index.ts +8 -3
  155. package/src/types/observability.ts +86 -2
  156. package/tsconfig.typecheck.json +18 -0
  157. package/dist/langfuseApp.d.ts +0 -9
  158. package/dist/langfuseApp.js +0 -53
  159. package/dist/langfuseCache.d.ts +0 -15
  160. package/dist/langfuseCache.js +0 -54
  161. package/dist/langfuseClient.d.ts +0 -6
  162. package/dist/langfuseClient.js +0 -34
  163. package/dist/langfusePrompts.d.ts +0 -12
  164. package/dist/langfusePrompts.js +0 -89
  165. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  166. package/dist/langfuseRoutesEvaluations.js +0 -32
  167. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  168. package/dist/langfuseRoutesMiddleware.js +0 -13
  169. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  170. package/dist/langfuseRoutesPlayground.js +0 -36
  171. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  172. package/dist/langfuseRoutesPrompts.js +0 -56
  173. package/dist/langfuseRoutesTraces.d.ts +0 -2
  174. package/dist/langfuseRoutesTraces.js +0 -30
  175. package/dist/langfuseTracing.d.ts +0 -4
  176. package/dist/langfuseTracing.js +0 -32
  177. package/dist/langfuseTypes.d.ts +0 -96
  178. package/dist/langfuseTypes.js +0 -2
  179. package/dist/langfuseVercelAi.d.ts +0 -12
  180. package/dist/langfuseVercelAi.js +0 -49
  181. package/src/isolated/langfuseClient.isolated.ts +0 -60
  182. package/src/langfuseApp.test.ts +0 -135
  183. package/src/langfuseApp.ts +0 -62
  184. package/src/langfuseCache.test.ts +0 -75
  185. package/src/langfuseCache.ts +0 -72
  186. package/src/langfuseClient.test.ts +0 -69
  187. package/src/langfuseClient.ts +0 -34
  188. package/src/langfusePrompts.test.ts +0 -188
  189. package/src/langfusePrompts.ts +0 -121
  190. package/src/langfuseRoutes.test.ts +0 -256
  191. package/src/langfuseRoutesEvaluations.ts +0 -48
  192. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  193. package/src/langfuseRoutesMiddleware.ts +0 -12
  194. package/src/langfuseRoutesPlayground.ts +0 -50
  195. package/src/langfuseRoutesPrompts.ts +0 -87
  196. package/src/langfuseRoutesTraces.ts +0 -40
  197. package/src/langfuseTracing.test.ts +0 -68
  198. package/src/langfuseTracing.ts +0 -33
  199. package/src/langfuseTypes.ts +0 -103
  200. package/src/langfuseVercelAi.test.ts +0 -79
  201. package/src/langfuseVercelAi.ts +0 -72
@@ -0,0 +1,174 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.summarizeDatasetJudgments = exports.loadCurrentHumanAnswers = exports.loadDatasetItemSubjects = exports.writeHumanAnswer = exports.toHumanAnswerViews = void 0;
7
+ const api_1 = require("@terreno/api");
8
+ const luxon_1 = require("luxon");
9
+ const mongoose_1 = __importDefault(require("mongoose"));
10
+ const questionFingerprint_1 = require("../questionFingerprint");
11
+ const obsDatasetItem_1 = require("./models/obsDatasetItem");
12
+ const obsEvaluator_1 = require("./models/obsEvaluator");
13
+ const obsReviewBatch_1 = require("./models/obsReviewBatch");
14
+ const obsTrace_1 = require("./models/obsTrace");
15
+ const toObjectId = (value, label) => {
16
+ if (!mongoose_1.default.Types.ObjectId.isValid(String(value))) {
17
+ throw new api_1.APIError({ status: 400, title: `${label} must be an object id` });
18
+ }
19
+ return new mongoose_1.default.Types.ObjectId(String(value));
20
+ };
21
+ const toHumanAnswerViews = (answers) => {
22
+ if (!answers) {
23
+ return undefined;
24
+ }
25
+ return Object.fromEntries(Object.entries(answers).map(([evaluatorId, answer]) => {
26
+ return [
27
+ evaluatorId,
28
+ {
29
+ answeredAt: luxon_1.DateTime.fromJSDate(answer.answeredAt).toUTC().toISO() ?? "",
30
+ fingerprint: answer.fingerprint,
31
+ reviewItemId: answer.reviewItemId,
32
+ scores: answer.scores,
33
+ userId: answer.userId,
34
+ },
35
+ ];
36
+ }));
37
+ };
38
+ exports.toHumanAnswerViews = toHumanAnswerViews;
39
+ /** Record (or replace) one question set's answers on a dataset item. */
40
+ const writeHumanAnswer = async (params) => {
41
+ const item = await (0, obsDatasetItem_1.registerObsDatasetItem)().findOneOrNone({ _id: params.datasetItemId });
42
+ if (!item) {
43
+ return;
44
+ }
45
+ const entry = {
46
+ answeredAt: luxon_1.DateTime.utc().toJSDate(),
47
+ fingerprint: (0, questionFingerprint_1.questionSetFingerprint)(params.evaluator.dimensions),
48
+ reviewItemId: params.reviewItemId,
49
+ scores: params.scores,
50
+ ...(params.userId ? { userId: params.userId } : {}),
51
+ };
52
+ item.humanAnswers = { ...(item.humanAnswers ?? {}), [String(params.evaluator._id)]: entry };
53
+ item.markModified("humanAnswers");
54
+ await item.save();
55
+ };
56
+ exports.writeHumanAnswer = writeHumanAnswer;
57
+ /** Subjects for dataset items, keyed by item id: the source trace when it still exists. */
58
+ const loadDatasetItemSubjects = async (items) => {
59
+ const traceIds = items.flatMap((item) => (item.sourceTraceId ? [item.sourceTraceId] : []));
60
+ const traces = traceIds.length > 0 ? await (0, obsTrace_1.registerObsTrace)().find({ _id: { $in: traceIds } }) : [];
61
+ const traceById = new Map(traces.map((trace) => {
62
+ return [String(trace._id), trace];
63
+ }));
64
+ return new Map(items.map((item) => {
65
+ const trace = item.sourceTraceId ? traceById.get(String(item.sourceTraceId)) : undefined;
66
+ const subject = trace
67
+ ? {
68
+ input: trace.input,
69
+ output: trace.output,
70
+ traceId: String(trace._id),
71
+ traceName: trace.name,
72
+ }
73
+ : { input: item.input, output: item.expectedOutput };
74
+ return [String(item._id), subject];
75
+ }));
76
+ };
77
+ exports.loadDatasetItemSubjects = loadDatasetItemSubjects;
78
+ /** Dataset items whose answers from this question set match its current questions. */
79
+ const loadCurrentHumanAnswers = async (params) => {
80
+ const questionSet = await (0, obsEvaluator_1.registerObsEvaluator)().findOneOrNone({
81
+ _id: toObjectId(params.questionSetId, "questionSetId"),
82
+ });
83
+ if (!questionSet) {
84
+ throw new api_1.APIError({ status: 404, title: "Unknown question set" });
85
+ }
86
+ const fingerprint = (0, questionFingerprint_1.questionSetFingerprint)(questionSet.dimensions);
87
+ const key = String(questionSet._id);
88
+ const rows = await (0, obsDatasetItem_1.registerObsDatasetItem)()
89
+ .find({ datasetId: toObjectId(params.datasetId, "datasetId") })
90
+ .sort({ _id: 1 });
91
+ const items = rows.flatMap((item) => {
92
+ const answer = item.humanAnswers?.[key];
93
+ return answer && answer.fingerprint === fingerprint ? [{ answer, item }] : [];
94
+ });
95
+ return { fingerprint, items, questionSet };
96
+ };
97
+ exports.loadCurrentHumanAnswers = loadCurrentHumanAnswers;
98
+ /**
99
+ * Per-dataset judgment summaries: one row per question set that answered any item or has a
100
+ * review batch built from the dataset.
101
+ */
102
+ const summarizeDatasetJudgments = async (datasetIds) => {
103
+ const result = new Map();
104
+ if (datasetIds.length === 0) {
105
+ return result;
106
+ }
107
+ const items = await (0, obsDatasetItem_1.registerObsDatasetItem)()
108
+ .find({ datasetId: { $in: datasetIds } })
109
+ .select({ datasetId: 1, humanAnswers: 1 });
110
+ const batches = await (0, obsReviewBatch_1.registerObsReviewBatch)()
111
+ .find({ datasetId: { $in: datasetIds } })
112
+ .select({ datasetId: 1, evaluatorId: 1 });
113
+ const evaluatorIdsByDataset = new Map();
114
+ const addEvaluator = (datasetId, evaluatorId) => {
115
+ const ids = evaluatorIdsByDataset.get(datasetId) ?? new Set();
116
+ ids.add(evaluatorId);
117
+ evaluatorIdsByDataset.set(datasetId, ids);
118
+ };
119
+ for (const item of items) {
120
+ for (const evaluatorId of Object.keys(item.humanAnswers ?? {})) {
121
+ addEvaluator(String(item.datasetId), evaluatorId);
122
+ }
123
+ }
124
+ for (const batch of batches) {
125
+ addEvaluator(String(batch.datasetId), String(batch.evaluatorId));
126
+ }
127
+ const allEvaluatorIds = [
128
+ ...new Set([...evaluatorIdsByDataset.values()].flatMap((ids) => [...ids])),
129
+ ]
130
+ .filter((id) => mongoose_1.default.Types.ObjectId.isValid(id))
131
+ .map((id) => new mongoose_1.default.Types.ObjectId(id));
132
+ const evaluators = await (0, obsEvaluator_1.registerObsEvaluator)().find({ _id: { $in: allEvaluatorIds } });
133
+ const evaluatorById = new Map(evaluators.map((evaluator) => {
134
+ return [
135
+ String(evaluator._id),
136
+ { fingerprint: (0, questionFingerprint_1.questionSetFingerprint)(evaluator.dimensions), name: evaluator.name },
137
+ ];
138
+ }));
139
+ for (const datasetId of datasetIds) {
140
+ const key = String(datasetId);
141
+ const datasetItems = items.filter((item) => String(item.datasetId) === key);
142
+ const summaries = [];
143
+ for (const evaluatorId of evaluatorIdsByDataset.get(key) ?? []) {
144
+ const evaluator = evaluatorById.get(evaluatorId);
145
+ if (!evaluator) {
146
+ continue;
147
+ }
148
+ let answered = 0;
149
+ let current = 0;
150
+ for (const item of datasetItems) {
151
+ const answer = item.humanAnswers?.[evaluatorId];
152
+ if (!answer) {
153
+ continue;
154
+ }
155
+ answered += 1;
156
+ if (answer.fingerprint === evaluator.fingerprint) {
157
+ current += 1;
158
+ }
159
+ }
160
+ summaries.push({
161
+ answered,
162
+ current,
163
+ evaluatorId,
164
+ evaluatorName: evaluator.name,
165
+ stale: answered - current,
166
+ total: datasetItems.length,
167
+ });
168
+ }
169
+ summaries.sort((left, right) => left.evaluatorName.localeCompare(right.evaluatorName));
170
+ result.set(key, summaries);
171
+ }
172
+ return result;
173
+ };
174
+ exports.summarizeDatasetJudgments = summarizeDatasetJudgments;
@@ -0,0 +1,56 @@
1
+ import type { EvaluatorDimension, ObsEvaluatorDocument } from "../../types/observability";
2
+ import { type EvaluatorAiClient } from "../evaluate";
3
+ import { type AgreementAnswerSet, type JudgeAgreement, type ScoreValue } from "../judgeAgreement";
4
+ export declare const DEFAULT_JUDGE_CONCURRENCY = 5;
5
+ /** What a judge sees for one item, and the ids that pair it with the human answer. */
6
+ export interface JudgeRunSubject {
7
+ datasetItemId?: string;
8
+ input?: unknown;
9
+ output?: unknown;
10
+ traceId?: string;
11
+ }
12
+ /** The judge prompt version to run: its output schema and, when pinned, its system text. */
13
+ export interface JudgeRunPrompt {
14
+ outputSchema: Record<string, unknown>;
15
+ /** Version body sent as the system prompt; unset sends the judge's prompt name instead. */
16
+ system?: string;
17
+ }
18
+ export interface JudgeRunItemResult {
19
+ confidence?: number;
20
+ datasetItemId?: string;
21
+ /** True when a human answered a compared question and the judge failed or answered differently. */
22
+ disagrees: boolean;
23
+ error?: string;
24
+ /** Index of the subject in the input list. */
25
+ index: number;
26
+ scores?: Record<string, ScoreValue>;
27
+ traceId?: string;
28
+ }
29
+ export interface JudgeRunResult {
30
+ agreement: JudgeAgreement;
31
+ results: JudgeRunItemResult[];
32
+ }
33
+ /** Agreement between human answers and (possibly partial) judge results from this pipeline. */
34
+ export declare const agreementForResults: (params: {
35
+ dimensions: EvaluatorDimension[];
36
+ humanAnswers: AgreementAnswerSet[];
37
+ judge: Pick<ObsEvaluatorDocument, "dimensions">;
38
+ results: JudgeRunItemResult[];
39
+ }) => JudgeAgreement;
40
+ /**
41
+ * Run one judge prompt version over every subject and compare its answers with the human
42
+ * answers for the same items. Review-batch judge runs and judge experiments both go through
43
+ * here, so judge errors and disagreements are scored the same way everywhere.
44
+ */
45
+ export declare const runJudgeAgainstHumans: (params: {
46
+ ai: EvaluatorAiClient;
47
+ concurrency?: number;
48
+ /** Question set dimensions the humans answered. */
49
+ dimensions: EvaluatorDimension[];
50
+ humanAnswers: AgreementAnswerSet[];
51
+ judge: Pick<ObsEvaluatorDocument, "dimensions" | "judgePromptName" | "name" | "type">;
52
+ /** Called once per subject in input order, never concurrently. */
53
+ onResult?: (result: JudgeRunItemResult) => Promise<void> | void;
54
+ prompt: JudgeRunPrompt;
55
+ subjects: JudgeRunSubject[];
56
+ }) => Promise<JudgeRunResult>;
@@ -0,0 +1,111 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.runJudgeAgainstHumans = exports.agreementForResults = exports.DEFAULT_JUDGE_CONCURRENCY = void 0;
4
+ const evaluate_1 = require("../evaluate");
5
+ const judgeAgreement_1 = require("../judgeAgreement");
6
+ exports.DEFAULT_JUDGE_CONCURRENCY = 5;
7
+ /** Agreement between human answers and (possibly partial) judge results from this pipeline. */
8
+ const agreementForResults = (params) => {
9
+ return (0, judgeAgreement_1.computeJudgeAgreement)({
10
+ dimensions: params.dimensions,
11
+ humanAnswers: params.humanAnswers,
12
+ judgeAnswers: params.results.map((result) => {
13
+ return {
14
+ datasetItemId: result.datasetItemId,
15
+ error: result.error,
16
+ scores: result.scores,
17
+ traceId: result.traceId,
18
+ };
19
+ }),
20
+ judgeKeys: params.judge.dimensions.map((dimension) => dimension.key),
21
+ });
22
+ };
23
+ exports.agreementForResults = agreementForResults;
24
+ // Run async work with a fixed number in flight, keeping input order in the result.
25
+ const mapWithConcurrency = async (items, limit, fn) => {
26
+ const results = new Array(items.length);
27
+ let cursor = 0;
28
+ const workers = Array.from({ length: Math.min(Math.max(limit, 1), items.length) }, async () => {
29
+ while (cursor < items.length) {
30
+ const index = cursor;
31
+ cursor += 1;
32
+ results[index] = await fn(items[index], index);
33
+ }
34
+ });
35
+ await Promise.all(workers);
36
+ return results;
37
+ };
38
+ /**
39
+ * Run one judge prompt version over every subject and compare its answers with the human
40
+ * answers for the same items. Review-batch judge runs and judge experiments both go through
41
+ * here, so judge errors and disagreements are scored the same way everywhere.
42
+ */
43
+ const runJudgeAgainstHumans = async (params) => {
44
+ const judgeKeys = params.judge.dimensions.map((dimension) => dimension.key);
45
+ const comparedDimensions = params.dimensions.filter((dimension) => {
46
+ return judgeKeys.includes(dimension.key);
47
+ });
48
+ const humanByKey = new Map(params.humanAnswers.map((answer) => {
49
+ return [(0, judgeAgreement_1.agreementAnswerKey)(answer), answer];
50
+ }));
51
+ // Results are reported in input order, so buffer finished ones until their turn comes.
52
+ const finished = new Array(params.subjects.length);
53
+ let nextToReport = 0;
54
+ let reporting = Promise.resolve();
55
+ const report = async () => {
56
+ while (nextToReport < finished.length && finished[nextToReport]) {
57
+ const result = finished[nextToReport];
58
+ nextToReport += 1;
59
+ await params.onResult?.(result);
60
+ }
61
+ };
62
+ const results = await mapWithConcurrency(params.subjects, params.concurrency ?? exports.DEFAULT_JUDGE_CONCURRENCY, async (subject, index) => {
63
+ const run = await (0, evaluate_1.runLlmJudgeEvaluator)({
64
+ evaluator: {
65
+ dimensions: params.judge.dimensions,
66
+ judgePromptName: params.judge.judgePromptName,
67
+ name: params.judge.name,
68
+ type: params.judge.type,
69
+ },
70
+ input: subject.input,
71
+ output: subject.output,
72
+ }, params.ai, params.prompt.outputSchema, params.prompt.system === undefined ? {} : { judgeSystemPrompt: params.prompt.system });
73
+ const ids = {
74
+ ...(subject.datasetItemId ? { datasetItemId: subject.datasetItemId } : {}),
75
+ ...(subject.traceId ? { traceId: subject.traceId } : {}),
76
+ };
77
+ const human = humanByKey.get((0, judgeAgreement_1.agreementAnswerKey)(ids));
78
+ // Same rule as computeJudgeAgreement: an errored judge answers nothing.
79
+ const disagrees = comparedDimensions.some((dimension) => {
80
+ const humanValue = human?.scores?.[dimension.key];
81
+ if (humanValue === undefined) {
82
+ return false;
83
+ }
84
+ const judgeValue = run.error ? undefined : run.scores?.[dimension.key];
85
+ return !(0, judgeAgreement_1.answersAgree)(dimension, humanValue, judgeValue);
86
+ });
87
+ const result = {
88
+ ...ids,
89
+ confidence: run.confidence,
90
+ disagrees,
91
+ error: run.error,
92
+ index,
93
+ scores: run.scores,
94
+ };
95
+ finished[index] = result;
96
+ reporting = reporting.then(report);
97
+ await reporting;
98
+ return result;
99
+ });
100
+ await reporting;
101
+ return {
102
+ agreement: (0, exports.agreementForResults)({
103
+ dimensions: params.dimensions,
104
+ humanAnswers: params.humanAnswers,
105
+ judge: params.judge,
106
+ results,
107
+ }),
108
+ results,
109
+ };
110
+ };
111
+ exports.runJudgeAgainstHumans = runJudgeAgainstHumans;
@@ -9,9 +9,11 @@ const obsDatasetItem_1 = require("./models/obsDatasetItem");
9
9
  const obsEvaluator_1 = require("./models/obsEvaluator");
10
10
  const obsExperiment_1 = require("./models/obsExperiment");
11
11
  const obsExperimentItem_1 = require("./models/obsExperimentItem");
12
+ const obsJudgeRun_1 = require("./models/obsJudgeRun");
12
13
  const obsPrompt_1 = require("./models/obsPrompt");
13
14
  const obsPromptLabel_1 = require("./models/obsPromptLabel");
14
15
  const obsPromptVersion_1 = require("./models/obsPromptVersion");
16
+ const obsReviewBatch_1 = require("./models/obsReviewBatch");
15
17
  const obsReviewItem_1 = require("./models/obsReviewItem");
16
18
  const obsScore_1 = require("./models/obsScore");
17
19
  const obsSpan_1 = require("./models/obsSpan");
@@ -27,6 +29,8 @@ const registerLocalObservabilityModels = () => {
27
29
  (0, obsScore_1.registerObsScore)();
28
30
  (0, obsEvaluator_1.registerObsEvaluator)();
29
31
  (0, obsReviewItem_1.registerObsReviewItem)();
32
+ (0, obsReviewBatch_1.registerObsReviewBatch)();
33
+ (0, obsJudgeRun_1.registerObsJudgeRun)();
30
34
  (0, obsDataset_1.registerObsDataset)();
31
35
  (0, obsDatasetItem_1.registerObsDatasetItem)();
32
36
  (0, obsExperiment_1.registerObsExperiment)();
@@ -25,6 +25,10 @@ const obsDatasetItemSchema = new mongoose_1.default.Schema({
25
25
  description: "Human- or machine-authored expected output label",
26
26
  type: mongoose_1.default.Schema.Types.Mixed,
27
27
  },
28
+ humanAnswers: {
29
+ description: "Human answers keyed by question-set evaluator id: {fingerprint, scores, reviewItemId, userId, answeredAt}. An answer is current while its fingerprint matches the question set",
30
+ type: mongoose_1.default.Schema.Types.Mixed,
31
+ },
28
32
  input: {
29
33
  description: "Dataset item input payload",
30
34
  required: true,
@@ -29,6 +29,10 @@ const obsEvaluatorSchema = new mongoose_1.default.Schema({
29
29
  type: String,
30
30
  },
31
31
  key: { description: "Dimension key written onto scores", required: true, type: String },
32
+ question: {
33
+ description: "Question shown to human reviewers for this dimension",
34
+ type: String,
35
+ },
32
36
  range: {
33
37
  description: "Numeric bounds, categorical values, or a free-text hint",
34
38
  type: String,
@@ -32,6 +32,18 @@ const obsExperimentSchema = new mongoose_1.default.Schema({
32
32
  description: "When true, synthetic/unproofread items are included",
33
33
  type: Boolean,
34
34
  },
35
+ judgeEvaluatorId: {
36
+ description: "Judge experiments: llm-judge evaluator whose prompt versions are compared",
37
+ ref: "ObsEvaluator",
38
+ type: mongoose_1.default.Schema.Types.ObjectId,
39
+ },
40
+ kind: {
41
+ default: "prompt",
42
+ description: "prompt compares product prompt versions; judge ranks judge prompt versions by agreement with human answers",
43
+ enum: ["judge", "prompt"],
44
+ required: true,
45
+ type: String,
46
+ },
35
47
  modelOverride: {
36
48
  description: "Optional model id override for all generations",
37
49
  type: String,
@@ -42,6 +54,15 @@ const obsExperimentSchema = new mongoose_1.default.Schema({
42
54
  required: true,
43
55
  type: String,
44
56
  },
57
+ questionSetId: {
58
+ description: "Judge experiments: human question set whose answers are the reference",
59
+ ref: "ObsEvaluator",
60
+ type: mongoose_1.default.Schema.Types.ObjectId,
61
+ },
62
+ questionsFingerprint: {
63
+ description: "Judge experiments: question-set fingerprint of the compared human answers",
64
+ type: String,
65
+ },
45
66
  results: {
46
67
  description: "Aggregated gate, outlier, and progress results",
47
68
  type: mongoose_1.default.Schema.Types.Mixed,
@@ -53,6 +74,10 @@ const obsExperimentSchema = new mongoose_1.default.Schema({
53
74
  required: true,
54
75
  type: String,
55
76
  },
77
+ targetAgreement: {
78
+ description: "Judge experiments: agreement rate (0–1) a judge version must reach to pass",
79
+ type: Number,
80
+ },
56
81
  thresholds: {
57
82
  default: [],
58
83
  description: "Gate thresholds evaluated after the run",
@@ -78,6 +103,9 @@ obsExperimentSchema.index({ created: -1, datasetId: 1 });
78
103
  obsExperimentSchema.index({ created: -1, status: 1 });
79
104
  // biome-ignore assist/source/useSortedKeys: promptName must be the leading key for prompt-filtered lists
80
105
  obsExperimentSchema.index({ promptName: 1, created: -1 });
106
+ // Evaluator calibration reads the latest judge experiment per judge evaluator.
107
+ // biome-ignore assist/source/useSortedKeys: judgeEvaluatorId must lead for calibration lookups
108
+ obsExperimentSchema.index({ judgeEvaluatorId: 1, created: -1 });
81
109
  const registerObsExperiment = () => {
82
110
  if (mongoose_1.default.models.ObsExperiment) {
83
111
  return mongoose_1.default.models.ObsExperiment;
@@ -24,6 +24,10 @@ const obsExperimentItemSchema = new mongoose_1.default.Schema({
24
24
  description: "When true, this row failed a gate or evaluator",
25
25
  type: Boolean,
26
26
  },
27
+ humanScores: {
28
+ description: "Judge experiments: human answers the judge versions are compared with",
29
+ type: mongoose_1.default.Schema.Types.Mixed,
30
+ },
27
31
  versionResults: {
28
32
  default: {},
29
33
  description: "Per-version outputs and evaluator score maps keyed by version number",
@@ -0,0 +1,2 @@
1
+ import type { ObsJudgeRunModel } from "../../../types/observability";
2
+ export declare const registerObsJudgeRun: () => ObsJudgeRunModel;
@@ -0,0 +1,60 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.registerObsJudgeRun = void 0;
7
+ const api_1 = require("@terreno/api");
8
+ const mongoose_1 = __importDefault(require("mongoose"));
9
+ const obsJudgeRunSchema = new mongoose_1.default.Schema({
10
+ batchId: {
11
+ description: "Review batch the judge answered",
12
+ ref: "ObsReviewBatch",
13
+ required: true,
14
+ type: mongoose_1.default.Schema.Types.ObjectId,
15
+ },
16
+ completedAt: { description: "When the judge finished every item", required: true, type: Date },
17
+ evaluatorId: {
18
+ description: "LLM-judge evaluator that produced the answers",
19
+ ref: "ObsEvaluator",
20
+ required: true,
21
+ type: mongoose_1.default.Schema.Types.ObjectId,
22
+ },
23
+ modelId: { description: "Model id of the AI client that ran the judge", type: String },
24
+ results: {
25
+ default: [],
26
+ description: "One judge answer set per batch trace",
27
+ type: [
28
+ {
29
+ confidence: { description: "Judge confidence in 0–1 when returned", type: Number },
30
+ datasetItemId: {
31
+ description: "Dataset item the judge answered, for dataset review batches",
32
+ ref: "ObsDatasetItem",
33
+ type: mongoose_1.default.Schema.Types.ObjectId,
34
+ },
35
+ error: { description: "Why the judge could not answer this item", type: String },
36
+ scores: {
37
+ description: "Dimension-keyed judge answers",
38
+ type: mongoose_1.default.Schema.Types.Mixed,
39
+ },
40
+ traceId: {
41
+ description: "Trace the judge answered, unset for dataset items without a trace",
42
+ ref: "ObsTrace",
43
+ type: mongoose_1.default.Schema.Types.ObjectId,
44
+ },
45
+ },
46
+ ],
47
+ },
48
+ }, { strict: "throw", toJSON: { virtuals: true }, toObject: { virtuals: true } });
49
+ obsJudgeRunSchema.plugin(api_1.createdUpdatedPlugin);
50
+ obsJudgeRunSchema.plugin(api_1.isDeletedPlugin);
51
+ obsJudgeRunSchema.plugin(api_1.findOneOrNone);
52
+ obsJudgeRunSchema.plugin(api_1.findExactlyOne);
53
+ obsJudgeRunSchema.index({ batchId: 1, completedAt: -1 });
54
+ const registerObsJudgeRun = () => {
55
+ if (mongoose_1.default.models.ObsJudgeRun) {
56
+ return mongoose_1.default.models.ObsJudgeRun;
57
+ }
58
+ return mongoose_1.default.model("ObsJudgeRun", obsJudgeRunSchema);
59
+ };
60
+ exports.registerObsJudgeRun = registerObsJudgeRun;
@@ -0,0 +1,2 @@
1
+ import type { ObsReviewBatchModel } from "../../../types/observability";
2
+ export declare const registerObsReviewBatch: () => ObsReviewBatchModel;
@@ -0,0 +1,48 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.registerObsReviewBatch = void 0;
7
+ const api_1 = require("@terreno/api");
8
+ const mongoose_1 = __importDefault(require("mongoose"));
9
+ const obsReviewBatchSchema = new mongoose_1.default.Schema({
10
+ datasetId: {
11
+ description: "Dataset the batch was built from, when reviewers judge dataset items",
12
+ ref: "ObsDataset",
13
+ type: mongoose_1.default.Schema.Types.ObjectId,
14
+ },
15
+ description: { description: "Operator note about why this batch exists", type: String },
16
+ evaluatorId: {
17
+ description: "Human evaluator whose questions reviewers answer for every item",
18
+ ref: "ObsEvaluator",
19
+ required: true,
20
+ type: mongoose_1.default.Schema.Types.ObjectId,
21
+ },
22
+ name: { description: "Operator-facing batch name", required: true, type: String },
23
+ promptName: {
24
+ description: "Prompt the batch traces were drawn from, when filtered by prompt",
25
+ type: String,
26
+ },
27
+ questionsFingerprint: {
28
+ description: "Question-set fingerprint when the batch was created",
29
+ type: String,
30
+ },
31
+ traceIds: {
32
+ default: [],
33
+ description: "Traces (one input and one output each) included in the batch",
34
+ type: [{ ref: "ObsTrace", type: mongoose_1.default.Schema.Types.ObjectId }],
35
+ },
36
+ }, { strict: "throw", toJSON: { virtuals: true }, toObject: { virtuals: true } });
37
+ obsReviewBatchSchema.plugin(api_1.createdUpdatedPlugin);
38
+ obsReviewBatchSchema.plugin(api_1.isDeletedPlugin);
39
+ obsReviewBatchSchema.plugin(api_1.findOneOrNone);
40
+ obsReviewBatchSchema.plugin(api_1.findExactlyOne);
41
+ obsReviewBatchSchema.index({ evaluatorId: 1 });
42
+ const registerObsReviewBatch = () => {
43
+ if (mongoose_1.default.models.ObsReviewBatch) {
44
+ return mongoose_1.default.models.ObsReviewBatch;
45
+ }
46
+ return mongoose_1.default.model("ObsReviewBatch", obsReviewBatchSchema);
47
+ };
48
+ exports.registerObsReviewBatch = registerObsReviewBatch;
@@ -12,6 +12,11 @@ const obsReviewItemSchema = new mongoose_1.default.Schema({
12
12
  ref: "User",
13
13
  type: mongoose_1.default.Schema.Types.ObjectId,
14
14
  },
15
+ batchId: {
16
+ description: "Review batch this item belongs to, when it was created as part of one",
17
+ ref: "ObsReviewBatch",
18
+ type: mongoose_1.default.Schema.Types.ObjectId,
19
+ },
15
20
  comment: { description: "Reviewer comment submitted with scores", type: String },
16
21
  datasetItemId: {
17
22
  description: "Optional dataset item this review is bound to",
@@ -51,9 +56,8 @@ const obsReviewItemSchema = new mongoose_1.default.Schema({
51
56
  type: String,
52
57
  },
53
58
  traceId: {
54
- description: "Trace under review",
59
+ description: "Trace under review. Unset when the item judges a dataset item that has no source trace",
55
60
  ref: "ObsTrace",
56
- required: true,
57
61
  type: mongoose_1.default.Schema.Types.ObjectId,
58
62
  },
59
63
  }, { strict: "throw", toJSON: { virtuals: true }, toObject: { virtuals: true } });
@@ -66,6 +70,10 @@ const reviewIndexSpec = [
66
70
  ["enqueuedAt", 1],
67
71
  ];
68
72
  obsReviewItemSchema.index(Object.fromEntries(reviewIndexSpec));
73
+ // Batch list, detail, and queue reads all scope by batchId, then by status.
74
+ obsReviewItemSchema.index({ batchId: 1, status: 1 });
75
+ // Dataset batches skip items that already wait in an open review for the same question set.
76
+ obsReviewItemSchema.index({ datasetItemId: 1, evaluatorId: 1 });
69
77
  const registerObsReviewItem = () => {
70
78
  if (mongoose_1.default.models.ObsReviewItem) {
71
79
  return mongoose_1.default.models.ObsReviewItem;
@@ -26,6 +26,10 @@ const obsTraceSchema = new mongoose_1.default.Schema({
26
26
  type: [
27
27
  // No subdocument _id, so `$addToSet` dedupes repeated refs from replayed phases.
28
28
  new mongoose_1.default.Schema({
29
+ includedBy: {
30
+ description: "Prompt whose reference pulled this version in; unset for a top-level prompt",
31
+ type: String,
32
+ },
29
33
  label: { description: "Label resolved at call time", type: String },
30
34
  name: { description: "Prompt name", required: true, type: String },
31
35
  version: { description: "Prompt version number", required: true, type: Number },
@@ -0,0 +1,31 @@
1
+ import type { ObsPromptRef } from "../../types/observability";
2
+ export interface PromptReference {
3
+ label?: string;
4
+ name: string;
5
+ tag: string;
6
+ version?: number;
7
+ }
8
+ export interface ResolvedPromptReference {
9
+ body: string;
10
+ label?: string;
11
+ version: number;
12
+ }
13
+ /** Loads the referenced prompt's raw body; undefined when the name, label, or version is unknown. */
14
+ export type PromptReferenceLookup = (reference: Omit<PromptReference, "tag">) => Promise<ResolvedPromptReference | undefined>;
15
+ export interface ResolvedPromptText {
16
+ /** Every prompt version pulled in, depth-first, once each, tagged with the prompt that included it. */
17
+ includes: ObsPromptRef[];
18
+ text: string;
19
+ }
20
+ /** Identity of one recorded include, for deduping. */
21
+ export declare const promptIncludeKey: (ref: ObsPromptRef) => string;
22
+ /**
23
+ * Replaces every reference in `text` with the referenced prompt's body, recursively. `ancestors`
24
+ * is the chain of prompt names already being resolved (start with the including prompt's own
25
+ * name), so a cycle or a chain deeper than MAX_PROMPT_REFERENCE_DEPTH fails with 400.
26
+ */
27
+ export declare const resolvePromptReferences: (params: {
28
+ ancestors: string[];
29
+ lookup: PromptReferenceLookup;
30
+ text: string;
31
+ }) => Promise<ResolvedPromptText>;