@terreno/ai 57.20.0 → 57.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. package/CHANGELOG.md +81 -0
  2. package/README.md +3 -3
  3. package/biome.jsonc +3 -2
  4. package/dist/harness/agentLoop.js +1 -0
  5. package/dist/harness/commit.d.ts +4 -3
  6. package/dist/harness/commit.js +23 -4
  7. package/dist/index.d.ts +6 -9
  8. package/dist/index.js +14 -25
  9. package/dist/observability/adminScreens.js +5 -1
  10. package/dist/observability/evaluate.d.ts +8 -1
  11. package/dist/observability/evaluate.js +10 -2
  12. package/dist/observability/judgeAgreement.d.ts +53 -0
  13. package/dist/observability/judgeAgreement.js +123 -0
  14. package/dist/observability/local/datasetStore.d.ts +5 -0
  15. package/dist/observability/local/datasetStore.js +7 -2
  16. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  17. package/dist/observability/local/evaluatorStore.js +73 -2
  18. package/dist/observability/local/experimentRunner.d.ts +42 -8
  19. package/dist/observability/local/experimentRunner.js +390 -11
  20. package/dist/observability/local/humanAnswers.d.ts +57 -0
  21. package/dist/observability/local/humanAnswers.js +174 -0
  22. package/dist/observability/local/judgeRun.d.ts +56 -0
  23. package/dist/observability/local/judgeRun.js +111 -0
  24. package/dist/observability/local/localPlugin.js +4 -0
  25. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  26. package/dist/observability/local/models/obsEvaluator.js +4 -0
  27. package/dist/observability/local/models/obsExperiment.js +28 -0
  28. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  29. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  30. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  31. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  32. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  33. package/dist/observability/local/models/obsReviewItem.js +10 -2
  34. package/dist/observability/local/models/obsTrace.js +4 -0
  35. package/dist/observability/local/promptReferences.d.ts +31 -0
  36. package/dist/observability/local/promptReferences.js +114 -0
  37. package/dist/observability/local/promptStore.d.ts +55 -19
  38. package/dist/observability/local/promptStore.js +171 -7
  39. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  40. package/dist/observability/local/reviewBatchStore.js +517 -0
  41. package/dist/observability/local/reviewStore.d.ts +24 -3
  42. package/dist/observability/local/reviewStore.js +98 -22
  43. package/dist/observability/local/traceStore.d.ts +14 -1
  44. package/dist/observability/local/traceStore.js +144 -11
  45. package/dist/observability/observabilityApp.js +10 -1
  46. package/dist/observability/questionFingerprint.d.ts +7 -0
  47. package/dist/observability/questionFingerprint.js +23 -0
  48. package/dist/observability/routes/experiments.js +12 -1
  49. package/dist/observability/routes/review.js +3 -1
  50. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  51. package/dist/observability/routes/reviewBatches.js +78 -0
  52. package/dist/observability/routes/traces.d.ts +1 -1
  53. package/dist/observability/routes/traces.js +34 -2
  54. package/dist/observability/types.d.ts +10 -1
  55. package/dist/service/aiService.js +4 -1
  56. package/dist/service/chatTurn.js +38 -31
  57. package/dist/types/harness.d.ts +10 -4
  58. package/dist/types/index.d.ts +8 -3
  59. package/dist/types/observability.d.ts +74 -2
  60. package/e2e/smoke.ts +227 -0
  61. package/package.json +14 -28
  62. package/src/aiApp.test.ts +22 -17
  63. package/src/harness/agentLoop.test.ts +0 -3
  64. package/src/harness/agentLoop.ts +2 -1
  65. package/src/harness/agentRecovery.test.ts +30 -30
  66. package/src/harness/approvals.test.ts +11 -9
  67. package/src/harness/commit.ts +24 -4
  68. package/src/harness/events.test.ts +10 -6
  69. package/src/harness/extensions.test.ts +6 -8
  70. package/src/harness/ownership.test.ts +20 -9
  71. package/src/harness/retries.test.ts +6 -3
  72. package/src/harness/routes/conversations.test.ts +2 -2
  73. package/src/harness/routes/tasks.test.ts +2 -2
  74. package/src/harness/runners/jobsRunner.test.ts +1 -1
  75. package/src/harness/subagent.test.ts +29 -4
  76. package/src/harness/waits.test.ts +6 -4
  77. package/src/index.test.ts +3 -20
  78. package/src/index.ts +23 -26
  79. package/src/isolated/gptStream.isolated.ts +25 -28
  80. package/src/isolated/lazyProjectModel.isolated.ts +2 -2
  81. package/src/models/gptHistory.test.ts +10 -6
  82. package/src/observability/adminScreens.test.ts +7 -0
  83. package/src/observability/adminScreens.ts +6 -1
  84. package/src/observability/evaluate.test.ts +12 -10
  85. package/src/observability/evaluate.ts +12 -2
  86. package/src/observability/judgeAgreement.test.ts +111 -0
  87. package/src/observability/judgeAgreement.ts +184 -0
  88. package/src/observability/local/datasetStore.ts +30 -11
  89. package/src/observability/local/evaluatorStore.ts +96 -2
  90. package/src/observability/local/experimentRunner.test.ts +9 -12
  91. package/src/observability/local/experimentRunner.ts +497 -17
  92. package/src/observability/local/humanAnswers.ts +233 -0
  93. package/src/observability/local/judgePipeline.test.ts +444 -0
  94. package/src/observability/local/judgeRun.test.ts +173 -0
  95. package/src/observability/local/judgeRun.ts +184 -0
  96. package/src/observability/local/localPlugin.ts +4 -0
  97. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  98. package/src/observability/local/models/obsEvaluator.ts +4 -0
  99. package/src/observability/local/models/obsExperiment.ts +29 -0
  100. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  101. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  102. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  103. package/src/observability/local/models/obsReviewItem.ts +11 -2
  104. package/src/observability/local/models/obsTrace.ts +5 -0
  105. package/src/observability/local/obsModels.test.ts +13 -0
  106. package/src/observability/local/promptReferences.test.ts +128 -0
  107. package/src/observability/local/promptReferences.ts +149 -0
  108. package/src/observability/local/promptStore.test.ts +295 -3
  109. package/src/observability/local/promptStore.ts +243 -32
  110. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  111. package/src/observability/local/reviewBatchStore.ts +656 -0
  112. package/src/observability/local/reviewStore.ts +122 -25
  113. package/src/observability/local/traceStore.test.ts +139 -1
  114. package/src/observability/local/traceStore.ts +162 -11
  115. package/src/observability/observabilityApp.test.ts +13 -10
  116. package/src/observability/observabilityApp.ts +10 -1
  117. package/src/observability/questionFingerprint.test.ts +33 -0
  118. package/src/observability/questionFingerprint.ts +24 -0
  119. package/src/observability/routes/datasets.test.ts +2 -2
  120. package/src/observability/routes/evaluators.test.ts +2 -2
  121. package/src/observability/routes/experiments.test.ts +39 -2
  122. package/src/observability/routes/experiments.ts +13 -1
  123. package/src/observability/routes/prompts.test.ts +3 -3
  124. package/src/observability/routes/review.test.ts +2 -2
  125. package/src/observability/routes/review.ts +3 -1
  126. package/src/observability/routes/reviewBatches.test.ts +209 -0
  127. package/src/observability/routes/reviewBatches.ts +138 -0
  128. package/src/observability/routes/status.test.ts +8 -8
  129. package/src/observability/routes/testMultiStage.test.ts +6 -6
  130. package/src/observability/routes/traces.test.ts +63 -2
  131. package/src/observability/routes/traces.ts +46 -4
  132. package/src/observability/types.ts +5 -2
  133. package/src/routes/files.test.ts +5 -3
  134. package/src/routes/gpt.test.ts +21 -14
  135. package/src/routes/gptActions.test.ts +7 -8
  136. package/src/routes/gptDatasets.test.ts +1 -1
  137. package/src/routes/gptFiles.test.ts +1 -1
  138. package/src/routes/gptHistories.test.ts +15 -9
  139. package/src/routes/mcp.test.ts +3 -3
  140. package/src/routes/projects.test.ts +5 -3
  141. package/src/routes/routes.test.ts +156 -115
  142. package/src/service/aiService.test.ts +22 -1
  143. package/src/service/aiService.ts +4 -1
  144. package/src/service/asks.test.ts +14 -6
  145. package/src/service/chatTurn.ts +42 -33
  146. package/src/service/fileStorage.test.ts +5 -2
  147. package/src/service/getMCPTools.test.ts +9 -5
  148. package/src/service/mcpService.test.ts +6 -3
  149. package/src/tests/bunSetup.ts +0 -54
  150. package/src/tests/chatHarness.ts +4 -4
  151. package/src/tests/fakeJudge.ts +8 -0
  152. package/src/tests/helpers.ts +26 -4
  153. package/src/types/harness.ts +11 -4
  154. package/src/types/index.ts +8 -3
  155. package/src/types/observability.ts +86 -2
  156. package/tsconfig.typecheck.json +18 -0
  157. package/dist/langfuseApp.d.ts +0 -9
  158. package/dist/langfuseApp.js +0 -53
  159. package/dist/langfuseCache.d.ts +0 -15
  160. package/dist/langfuseCache.js +0 -54
  161. package/dist/langfuseClient.d.ts +0 -6
  162. package/dist/langfuseClient.js +0 -34
  163. package/dist/langfusePrompts.d.ts +0 -12
  164. package/dist/langfusePrompts.js +0 -89
  165. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  166. package/dist/langfuseRoutesEvaluations.js +0 -32
  167. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  168. package/dist/langfuseRoutesMiddleware.js +0 -13
  169. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  170. package/dist/langfuseRoutesPlayground.js +0 -36
  171. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  172. package/dist/langfuseRoutesPrompts.js +0 -56
  173. package/dist/langfuseRoutesTraces.d.ts +0 -2
  174. package/dist/langfuseRoutesTraces.js +0 -30
  175. package/dist/langfuseTracing.d.ts +0 -4
  176. package/dist/langfuseTracing.js +0 -32
  177. package/dist/langfuseTypes.d.ts +0 -96
  178. package/dist/langfuseTypes.js +0 -2
  179. package/dist/langfuseVercelAi.d.ts +0 -12
  180. package/dist/langfuseVercelAi.js +0 -49
  181. package/src/isolated/langfuseClient.isolated.ts +0 -60
  182. package/src/langfuseApp.test.ts +0 -135
  183. package/src/langfuseApp.ts +0 -62
  184. package/src/langfuseCache.test.ts +0 -75
  185. package/src/langfuseCache.ts +0 -72
  186. package/src/langfuseClient.test.ts +0 -69
  187. package/src/langfuseClient.ts +0 -34
  188. package/src/langfusePrompts.test.ts +0 -188
  189. package/src/langfusePrompts.ts +0 -121
  190. package/src/langfuseRoutes.test.ts +0 -256
  191. package/src/langfuseRoutesEvaluations.ts +0 -48
  192. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  193. package/src/langfuseRoutesMiddleware.ts +0 -12
  194. package/src/langfuseRoutesPlayground.ts +0 -50
  195. package/src/langfuseRoutesPrompts.ts +0 -87
  196. package/src/langfuseRoutesTraces.ts +0 -40
  197. package/src/langfuseTracing.test.ts +0 -68
  198. package/src/langfuseTracing.ts +0 -33
  199. package/src/langfuseTypes.ts +0 -103
  200. package/src/langfuseVercelAi.test.ts +0 -79
  201. package/src/langfuseVercelAi.ts +0 -72
package/package.json CHANGED
@@ -6,11 +6,7 @@
6
6
  "dependencies": {
7
7
  "@ai-sdk/mcp": "^1.0.21",
8
8
  "@google-cloud/storage": "^7.15.0",
9
- "@langfuse/client": "^5.0.1",
10
- "@langfuse/otel": "^5.0.1",
11
- "@opentelemetry/api": "^1.9.0",
12
- "@opentelemetry/sdk-node": "0.217.0",
13
- "@terreno/blocks": "57.20.0",
9
+ "@terreno/blocks": "57.22.0",
14
10
  "ai": "^6.0.97",
15
11
  "ajv": "8.20.0",
16
12
  "csv-parse": "^6.1.0",
@@ -22,10 +18,10 @@
22
18
  "description": "Provider-agnostic AI service with streaming chat, request logging, and admin tools — part of Terreno, the batteries-included TypeScript framework for universal apps.",
23
19
  "devDependencies": {
24
20
  "@biomejs/biome": "^2.3.6",
25
- "@terreno/admin-backend": "57.20.0",
26
- "@terreno/api": "57.20.0",
27
- "@terreno/jobs": "57.20.0",
28
- "@terreno/test": "57.20.0",
21
+ "@terreno/admin-backend": "57.22.0",
22
+ "@terreno/api": "57.22.0",
23
+ "@terreno/jobs": "57.22.0",
24
+ "@terreno/test": "57.22.0",
29
25
  "@types/bun": "^1.4.0",
30
26
  "@types/express": "^5.0.6",
31
27
  "@types/luxon": "^3.7.1",
@@ -87,18 +83,6 @@
87
83
  "require": "./dist/harness/runners/jobsRunner.js",
88
84
  "types": "./dist/harness/runners/jobsRunner.d.ts"
89
85
  },
90
- "./langfuseApp": {
91
- "default": "./dist/langfuseApp.js",
92
- "import": "./dist/langfuseApp.js",
93
- "require": "./dist/langfuseApp.js",
94
- "types": "./dist/langfuseApp.d.ts"
95
- },
96
- "./langfuseClient": {
97
- "default": "./dist/langfuseClient.js",
98
- "import": "./dist/langfuseClient.js",
99
- "require": "./dist/langfuseClient.js",
100
- "types": "./dist/langfuseClient.d.ts"
101
- },
102
86
  "./observability/localPlugin": {
103
87
  "default": "./dist/observability/local/localPlugin.js",
104
88
  "import": "./dist/observability/local/localPlugin.js",
@@ -136,8 +120,8 @@
136
120
  "name": "@terreno/ai",
137
121
  "peerDependencies": {
138
122
  "@ai-sdk/google-vertex": "^4.0.142",
139
- "@terreno/api": "57.20.0",
140
- "@terreno/jobs": "57.20.0",
123
+ "@terreno/api": "57.22.0",
124
+ "@terreno/jobs": "57.22.0",
141
125
  "mongoose": "^8.0.0 || ^9.0.0"
142
126
  },
143
127
  "peerDependenciesMeta": {
@@ -159,14 +143,16 @@
159
143
  "compile": "bun tsc",
160
144
  "compile:watch": "bun tsc -w",
161
145
  "dev": "bun tsc -w",
162
- "lint": "biome check ./src",
163
- "lint:fix": "biome check --write ./src",
164
- "lint:unsafefix": "biome check --fix --unsafe ./src",
146
+ "lint": "biome check ./src ./e2e",
147
+ "lint:fix": "biome check --write ./src ./e2e",
148
+ "lint:unsafefix": "biome check --fix --unsafe ./src ./e2e",
149
+ "smoke": "bun e2e/smoke.ts",
165
150
  "test": "bun test --preload ./src/tests/bunSetup.ts && for f in ./src/isolated/*.isolated.ts; do bun test --preload ./src/tests/bunSetup.ts \"$f\" || exit 1; done",
166
151
  "test:agent": "AGENT=1 bun test --only-failures --preload ./src/tests/bunSetup.ts && for f in ./src/isolated/*.isolated.ts; do AGENT=1 bun test --only-failures --preload ./src/tests/bunSetup.ts \"$f\" || exit 1; done",
167
152
  "test:ci": "bun test --preload ./src/tests/bunSetup.ts && for f in ./src/isolated/*.isolated.ts; do bun test --preload ./src/tests/bunSetup.ts \"$f\" || exit 1; done",
168
- "test:coverage": "bun run ../scripts/check-coverage.ts"
153
+ "test:coverage": "bun run ../scripts/check-coverage.ts",
154
+ "typecheck": "tsc -p tsconfig.typecheck.json"
169
155
  },
170
156
  "types": "dist/index.d.ts",
171
- "version": "57.20.0"
157
+ "version": "57.22.0"
172
158
  }
package/src/aiApp.test.ts CHANGED
@@ -10,7 +10,12 @@ import {GptHistory} from "./models/gptHistory";
10
10
  import type {FileStorageService} from "./service/fileStorage";
11
11
  import type {MCPService} from "./service/mcpService";
12
12
  import {createScriptedModel, modelCall, textStep, toolCallStep} from "./tests/chatHarness";
13
- import {authAsUserWithCredentials, ensureTestUsers, UserModel} from "./tests/helpers";
13
+ import {
14
+ authAsUserWithCredentials,
15
+ ensureTestUsers,
16
+ terrenoUserModel,
17
+ UserModel,
18
+ } from "./tests/helpers";
14
19
 
15
20
  const AI_APP_TEST_USER = {
16
21
  admin: false,
@@ -61,13 +66,13 @@ describe("AiApp", () => {
61
66
  };
62
67
 
63
68
  it("registers gpt and project routes by default", async () => {
64
- const {AIService} = await import("./service/aiService");
69
+ const {AIService} = await import("./service/aiService.js");
65
70
  const aiService = new AIService({model: createMockModel() as unknown as LanguageModel});
66
71
  const plugin = new AiApp({aiService});
67
72
  const app = new TerrenoApp({
68
73
  configureApp: (router) => plugin.register(router as unknown as express.Application),
69
74
  skipListen: true,
70
- userModel: UserModel,
75
+ userModel: terrenoUserModel,
71
76
  }).build();
72
77
 
73
78
  const agent = await authAsUser(app);
@@ -80,13 +85,13 @@ describe("AiApp", () => {
80
85
  });
81
86
 
82
87
  it("skips the project routes with projects: false", async () => {
83
- const {AIService} = await import("./service/aiService");
88
+ const {AIService} = await import("./service/aiService.js");
84
89
  const aiService = new AIService({model: createMockModel() as unknown as LanguageModel});
85
90
  const plugin = new AiApp({aiService, projects: false});
86
91
  const app = new TerrenoApp({
87
92
  configureApp: (router) => plugin.register(router as unknown as express.Application),
88
93
  skipListen: true,
89
- userModel: UserModel,
94
+ userModel: terrenoUserModel,
90
95
  }).build();
91
96
 
92
97
  const agent = await authAsUser(app);
@@ -95,14 +100,14 @@ describe("AiApp", () => {
95
100
  });
96
101
 
97
102
  it("passes asks to the gpt routes, so the model is offered the ask tools", async () => {
98
- const {AIService} = await import("./service/aiService");
103
+ const {AIService} = await import("./service/aiService.js");
99
104
  const model = createMockModel();
100
105
  const aiService = new AIService({model: model as unknown as LanguageModel});
101
106
  const plugin = new AiApp({aiService, asks: true});
102
107
  const app = new TerrenoApp({
103
108
  configureApp: (router) => plugin.register(router as unknown as express.Application),
104
109
  skipListen: true,
105
- userModel: UserModel,
110
+ userModel: terrenoUserModel,
106
111
  }).build();
107
112
  const agent = await authAsUser(app);
108
113
 
@@ -120,7 +125,7 @@ describe("AiApp", () => {
120
125
  });
121
126
 
122
127
  it("adds the documented headless turn actions to the history routes", async () => {
123
- const {AIService} = await import("./service/aiService");
128
+ const {AIService} = await import("./service/aiService.js");
124
129
  const model = createMockModel();
125
130
  const aiService = new AIService({model: model as unknown as LanguageModel});
126
131
  const app = new TerrenoApp({
@@ -129,7 +134,7 @@ describe("AiApp", () => {
129
134
  router as unknown as express.Application
130
135
  ),
131
136
  skipListen: true,
132
- userModel: UserModel,
137
+ userModel: terrenoUserModel,
133
138
  }).build();
134
139
  const agent = await authAsUser(app);
135
140
 
@@ -159,7 +164,7 @@ describe("AiApp", () => {
159
164
  });
160
165
 
161
166
  it("adds the headless actions only with asks on, so hosts without asks get no new endpoints", async () => {
162
- const {AIService} = await import("./service/aiService");
167
+ const {AIService} = await import("./service/aiService.js");
163
168
  const historyRoutes = async (asks?: boolean) => {
164
169
  const model = createMockModel();
165
170
  const aiService = new AIService({model: model as unknown as LanguageModel});
@@ -169,7 +174,7 @@ describe("AiApp", () => {
169
174
  router as unknown as express.Application
170
175
  ),
171
176
  skipListen: true,
172
- userModel: UserModel,
177
+ userModel: terrenoUserModel,
173
178
  }).build();
174
179
  const agent = await authAsUser(app);
175
180
  const openApi = await agent.get("/openapi.json");
@@ -226,7 +231,7 @@ describe("AiApp", () => {
226
231
  const app = new TerrenoApp({
227
232
  configureApp: (router) => plugin.register(router as unknown as express.Application),
228
233
  skipListen: true,
229
- userModel: UserModel,
234
+ userModel: terrenoUserModel,
230
235
  }).build();
231
236
  // Unauthenticated access to upload route: ensure it exists (returns 401, not 404).
232
237
  const upload = await supertest(app).post("/files/upload");
@@ -234,8 +239,8 @@ describe("AiApp", () => {
234
239
  });
235
240
 
236
241
  it("caps uploads at asks.maxFileSizeBytes and loads a files answer's uploads from storage", async () => {
237
- const {AIService} = await import("./service/aiService");
238
- const {FileAttachment} = await import("./models/fileAttachment");
242
+ const {AIService} = await import("./service/aiService.js");
243
+ const {FileAttachment} = await import("./models/fileAttachment.js");
239
244
  const text = Buffer.from("Coffee, 4.50\n");
240
245
  const upload = mock(async () => {
241
246
  const [user] = await UserModel.find({email: AI_APP_TEST_USER.email});
@@ -278,7 +283,7 @@ describe("AiApp", () => {
278
283
  const app = new TerrenoApp({
279
284
  configureApp: (router) => plugin.register(router as unknown as express.Application),
280
285
  skipListen: true,
281
- userModel: UserModel,
286
+ userModel: terrenoUserModel,
282
287
  }).build();
283
288
  const agent = await authAsUser(app);
284
289
 
@@ -329,7 +334,7 @@ describe("AiApp", () => {
329
334
  const app = new TerrenoApp({
330
335
  configureApp: (router) => plugin.register(router as unknown as express.Application),
331
336
  skipListen: true,
332
- userModel: UserModel,
337
+ userModel: terrenoUserModel,
333
338
  }).build();
334
339
  const upload = await supertest(app).post("/files/upload");
335
340
  expect(upload.status).toBe(404);
@@ -345,7 +350,7 @@ describe("AiApp", () => {
345
350
  const app = new TerrenoApp({
346
351
  configureApp: (router) => plugin.register(router as unknown as express.Application),
347
352
  skipListen: true,
348
- userModel: UserModel,
353
+ userModel: terrenoUserModel,
349
354
  }).build();
350
355
  const agent = await authAsUser(app);
351
356
  const servers = await agent.get("/mcp/servers");
@@ -178,9 +178,6 @@ const spansOf = (task: HarnessTaskDocument) =>
178
178
 
179
179
  describe("Agent turns", () => {
180
180
  beforeAll(async () => {
181
- // The shared preload's beforeEach imports this lazily under a 5 s hook timeout; a
182
- // cold first import can exceed it, so load it here first.
183
- await import("../langfuseClient");
184
181
  createLocalObservabilityPlugin();
185
182
  SpanModel = registerObsSpan();
186
183
  TraceModel = registerObsTrace();
@@ -653,7 +653,8 @@ export const createAgentTasks = (context: AgentLoopContext): AgentTasks => {
653
653
  }
654
654
  let returned: unknown;
655
655
  try {
656
- returned = await tool.execute(args.data, {
656
+ // `args.data` was parsed by this tool's own `parameters`, which match its `execute` args.
657
+ returned = await tool.execute(args.data as never, {
657
658
  conversationId,
658
659
  env: rt.env,
659
660
  output: (text: string) => {
@@ -44,38 +44,41 @@ type Reply = {
44
44
  };
45
45
 
46
46
  /** LanguageModelV2 mock answering each request with the next reply; `hang` blocks it first. */
47
- const scriptedModel = (replies: Reply[]) =>
48
- withGenerateStreaming({
49
- doGenerate: mock(async () => {
50
- const reply = replies.shift();
51
- if (!reply) {
52
- throw new Error("script exhausted");
53
- }
54
- reply.started?.release();
55
- if (reply.hang) {
56
- await reply.hang.promise;
57
- }
58
- return {
59
- content: reply.toolCall
60
- ? [
61
- {
62
- input: "{}",
63
- toolCallId: reply.toolCall.id,
64
- toolName: reply.toolCall.name,
65
- type: "tool-call" as const,
66
- },
67
- ]
68
- : [{text: reply.text ?? "", type: "text" as const}],
69
- finishReason: reply.toolCall ? ("tool-calls" as const) : ("stop" as const),
70
- usage: {inputTokens: 1, outputTokens: 1, totalTokens: 2},
71
- warnings: [],
72
- };
73
- }),
47
+ const scriptedModel = (replies: Reply[]) => {
48
+ // Declared before the call so `withGenerateStreaming` infers the mock's own type.
49
+ const doGenerate = mock(async () => {
50
+ const reply = replies.shift();
51
+ if (!reply) {
52
+ throw new Error("script exhausted");
53
+ }
54
+ reply.started?.release();
55
+ if (reply.hang) {
56
+ await reply.hang.promise;
57
+ }
58
+ return {
59
+ content: reply.toolCall
60
+ ? [
61
+ {
62
+ input: "{}",
63
+ toolCallId: reply.toolCall.id,
64
+ toolName: reply.toolCall.name,
65
+ type: "tool-call" as const,
66
+ },
67
+ ]
68
+ : [{text: reply.text ?? "", type: "text" as const}],
69
+ finishReason: reply.toolCall ? ("tool-calls" as const) : ("stop" as const),
70
+ usage: {inputTokens: 1, outputTokens: 1, totalTokens: 2},
71
+ warnings: [],
72
+ };
73
+ });
74
+ return withGenerateStreaming({
75
+ doGenerate,
74
76
  modelId: "mock-model",
75
77
  provider: "mock",
76
78
  specificationVersion: "v2" as const,
77
79
  supportedUrls: {},
78
80
  });
81
+ };
79
82
 
80
83
  interface ProcessHandle {
81
84
  die: () => void;
@@ -167,9 +170,6 @@ const toolMessages = async (conversationId: string) => {
167
170
 
168
171
  describe("Agent turns across a crash", () => {
169
172
  beforeAll(async () => {
170
- // The shared preload's beforeEach imports this lazily under a 5 s hook timeout; a
171
- // cold first import can exceed it, so load it here first.
172
- await import("../langfuseClient");
173
173
  createLocalObservabilityPlugin();
174
174
  SpanModel = registerObsSpan();
175
175
  TraceModel = registerObsTrace();
@@ -21,10 +21,11 @@ import {registerObsSpan} from "../observability/local/models/obsSpan";
21
21
  import {registerObsTrace} from "../observability/local/models/obsTrace";
22
22
  import {withGenerateStreaming} from "../tests/generateStream";
23
23
  import {harnessErrorMatching} from "../tests/harnessErrors";
24
- import {authAsUser, ensureTestUsers, UserModel} from "../tests/helpers";
24
+ import {authAsUser, ensureTestUsers, terrenoUserModel, UserModel} from "../tests/helpers";
25
25
  import type {
26
26
  HarnessApprovalDocument,
27
27
  HarnessApprovalResult,
28
+ HarnessTaskDefinition,
28
29
  HarnessTaskDocument,
29
30
  HarnessTestHooks,
30
31
  } from "../types/harness";
@@ -32,7 +33,6 @@ import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
32
33
  import type {HarnessModels} from "./commit";
33
34
  import {
34
35
  AGENT_TOOL_TASK_NAME,
35
- type AnyHarnessTaskDefinition,
36
36
  approvalGate,
37
37
  approvalTaskInput,
38
38
  defineAgent,
@@ -55,6 +55,9 @@ import {registerHarnessOwner} from "./models/harnessOwner";
55
55
  import {registerHarnessTask} from "./models/harnessTask";
56
56
  import {HarnessWaitRaceError, sendEventRecords} from "./waits";
57
57
 
58
+ /** A task definition whose input is not checked, so tests can create it with any input. */
59
+ type TestTaskDefinition = HarnessTaskDefinition<unknown, unknown, unknown>;
60
+
58
61
  // Whole tasks and turns on a real replica set; the first test also pays for index creation.
59
62
  setDefaultTimeout(30_000);
60
63
 
@@ -107,8 +110,7 @@ const statusIs = (id: unknown, status: string) => async (): Promise<boolean> =>
107
110
  const approvalOf = (taskId: unknown): Promise<HarnessApprovalDocument> =>
108
111
  ApprovalModel.findExactlyOne({taskId});
109
112
 
110
- const asAny = (definition: unknown): AnyHarnessTaskDefinition =>
111
- definition as AnyHarnessTaskDefinition;
113
+ const asAny = (definition: unknown): TestTaskDefinition => definition as TestTaskDefinition;
112
114
 
113
115
  const openHarness = async (
114
116
  registry: ReadonlyArray<HarnessRegistryEntry>,
@@ -138,7 +140,7 @@ const openHarness = async (
138
140
  };
139
141
 
140
142
  const buildApp = (harness: Harness): express.Application =>
141
- new TerrenoApp({skipListen: true, userModel: UserModel})
143
+ new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
142
144
  .register(new HarnessApp({harness}))
143
145
  .build();
144
146
 
@@ -176,7 +178,7 @@ const asksApproval = ({
176
178
  retry?: {maxAttempts: number};
177
179
  runs?: {count: number};
178
180
  timeout?: {hours: number};
179
- }): AnyHarnessTaskDefinition =>
181
+ }): TestTaskDefinition =>
180
182
  asAny(
181
183
  defineTask<{patientId: string}, unknown, HarnessApprovalResult>({
182
184
  approvals,
@@ -563,7 +565,7 @@ describe("Harness approvals", () => {
563
565
 
564
566
  describe("races and forged events", () => {
565
567
  /** Past the approval's timeout with the runner stopped, so the next owner resolves it. */
566
- const parkThenPassTimeout = async (definition: AnyHarnessTaskDefinition) => {
568
+ const parkThenPassTimeout = async (definition: TestTaskDefinition) => {
567
569
  const start = DateTime.now();
568
570
  Settings.now = () => start.toMillis();
569
571
  const first = await openHarness([definition]);
@@ -762,7 +764,7 @@ describe("Harness approvals", () => {
762
764
  });
763
765
 
764
766
  it("fails the task on rt.approval misuse and rejects invalid approval policies", async () => {
765
- const misuse = (options: unknown, key = "k"): AnyHarnessTaskDefinition =>
767
+ const misuse = (options: unknown, key = "k"): TestTaskDefinition =>
766
768
  asAny(
767
769
  defineTask<unknown, unknown, unknown>({
768
770
  initial: () => ({phase: "ask"}),
@@ -778,7 +780,7 @@ describe("Harness approvals", () => {
778
780
  version: 1,
779
781
  })
780
782
  );
781
- const cases: Array<[AnyHarnessTaskDefinition, string]> = [
783
+ const cases: Array<[TestTaskDefinition, string]> = [
782
784
  [misuse({title: "t"}, " "), "rt.approval requires a key"],
783
785
  [misuse({}), 'rt.approval("k") requires a title'],
784
786
  [misuse({summary: 3, title: "t"}), 'rt.approval("k") summary must be a string'],
@@ -152,9 +152,10 @@ const findByRequestId = async ({
152
152
  };
153
153
 
154
154
  /**
155
- * Prompt refs as stored on `ObsTrace.prompts` (`{label?, name, version}`; a registry
156
- * `body` is dropped), once each. Throws `invalidRequest` for a ref without a name, with a
157
- * non-string label, or without a positive integer version.
155
+ * Prompt refs as stored on `ObsTrace.prompts` (`{includedBy?, label?, name, version}`; a
156
+ * registry `body` is dropped and its `includes` follow it), once each. Throws
157
+ * `invalidRequest` for a ref without a name, with a non-string label or includedBy, with
158
+ * non-array includes, or without a positive integer version.
158
159
  */
159
160
  export const tracePromptRefs = (refs: HarnessPromptRef[] | undefined): HarnessPromptRef[] => {
160
161
  if (refs === undefined) {
@@ -164,7 +165,19 @@ export const tracePromptRefs = (refs: HarnessPromptRef[] | undefined): HarnessPr
164
165
  throw harnessError({detail: "prompts must be an array", kind: "invalidRequest"});
165
166
  }
166
167
  const unique = new Map<string, HarnessPromptRef>();
167
- for (const ref of refs) {
168
+ const expanded = refs.flatMap((ref) => {
169
+ if (ref?.includes !== undefined && !Array.isArray(ref.includes)) {
170
+ throw harnessError({
171
+ detail:
172
+ typeof ref.name === "string" && ref.name.trim()
173
+ ? `Prompt ref "${ref.name}" includes must be an array`
174
+ : "Prompt ref includes must be an array",
175
+ kind: "invalidRequest",
176
+ });
177
+ }
178
+ return [ref, ...(ref?.includes ?? [])];
179
+ });
180
+ for (const ref of expanded) {
168
181
  if (typeof ref?.name !== "string" || !ref.name.trim()) {
169
182
  throw harnessError({detail: "Each prompt ref needs a name", kind: "invalidRequest"});
170
183
  }
@@ -180,7 +193,14 @@ export const tracePromptRefs = (refs: HarnessPromptRef[] | undefined): HarnessPr
180
193
  kind: "invalidRequest",
181
194
  });
182
195
  }
196
+ if (ref.includedBy !== undefined && typeof ref.includedBy !== "string") {
197
+ throw harnessError({
198
+ detail: `Prompt ref "${ref.name}" includedBy must be a string`,
199
+ kind: "invalidRequest",
200
+ });
201
+ }
183
202
  const normalized = {
203
+ ...(ref.includedBy !== undefined ? {includedBy: ref.includedBy} : {}),
184
204
  ...(ref.label !== undefined ? {label: ref.label} : {}),
185
205
  name: ref.name,
186
206
  version: ref.version,
@@ -18,7 +18,7 @@ import {
18
18
  type SseClient,
19
19
  type SseFrame,
20
20
  } from "../tests/harnessStreaming";
21
- import {ensureTestUsers, UserModel} from "../tests/helpers";
21
+ import {ensureTestUsers, terrenoUserModel, UserModel} from "../tests/helpers";
22
22
  import type {HarnessStreamingOptions} from "../types/harness";
23
23
  import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
24
24
  import type {HarnessModels} from "./commit";
@@ -102,7 +102,9 @@ const serve = async (
102
102
  harness: Harness
103
103
  ): Promise<{app: express.Application; plugin: HarnessApp; url: string}> => {
104
104
  const plugin = new HarnessApp({harness, heartbeatInterval: {milliseconds: 50}});
105
- const app = new TerrenoApp({skipListen: true, userModel: UserModel}).register(plugin).build();
105
+ const app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
106
+ .register(plugin)
107
+ .build();
106
108
  const {stop, url} = await listen(app);
107
109
  cleanups.push(stop);
108
110
  return {app, plugin, url};
@@ -111,7 +113,8 @@ const serve = async (
111
113
  /** Open change-stream cursors on the server, idle ones included. */
112
114
  const openChangeStreamOps = async (): Promise<Array<Record<string, unknown>>> => {
113
115
  const dbName = mongoose.connection.db?.databaseName;
114
- return mongoose.connection.client
116
+ return mongoose.connection
117
+ .getClient()
115
118
  .db("admin")
116
119
  .aggregate([
117
120
  {$currentOp: {allUsers: true, idleCursors: true}},
@@ -168,10 +171,11 @@ const sse = async (
168
171
  /** Status and body of a request that must fail before streaming starts. */
169
172
  const refused = async (url: string, headers: Record<string, string> = {}) => {
170
173
  const response = await fetch(url, {headers});
171
- return {body: await response.json(), status: response.status};
174
+ return {body: (await response.json()) as {title?: string}, status: response.status};
172
175
  };
173
176
 
174
- const storedEvents = (streamId: unknown) => EventModel.find({streamId}).sort({seq: 1}).lean();
177
+ const storedEvents = (streamId: mongoose.Types.ObjectId | string) =>
178
+ EventModel.find({streamId}).sort({seq: 1}).lean();
175
179
 
176
180
  const deltaText = (frames: ReadonlyArray<SseFrame>): string =>
177
181
  frames
@@ -372,7 +376,7 @@ describe("Harness event stream", () => {
372
376
  });
373
377
 
374
378
  it("rejects a heartbeat interval that is not positive", () => {
375
- const app = new TerrenoApp({skipListen: true, userModel: UserModel});
379
+ const app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel});
376
380
  expect(() =>
377
381
  app
378
382
  .register(new HarnessApp({harness: {} as Harness, heartbeatInterval: {seconds: 0}}))
@@ -198,9 +198,6 @@ const llmSpans = (task: HarnessTaskDocument) =>
198
198
 
199
199
  describe("Extensions, hooks, wraps, and memos", () => {
200
200
  beforeAll(async () => {
201
- // The shared preload's beforeEach imports this lazily under a 5 s hook timeout; a
202
- // cold first import can exceed it, so load it here first.
203
- await import("../langfuseClient");
204
201
  createLocalObservabilityPlugin();
205
202
  SpanModel = registerObsSpan();
206
203
  TraceModel = registerObsTrace();
@@ -233,7 +230,7 @@ describe("Extensions, hooks, wraps, and memos", () => {
233
230
  expect(() => section("s", "x" as never)).toThrow(
234
231
  harnessErrorMatching("definitionInvalid", "section(s): build must be a function")
235
232
  );
236
- expect(() => hook("onEverything" as never, () => undefined)).toThrow(
233
+ expect(() => hook("onEverything" as never, (() => undefined) as never)).toThrow(
237
234
  harnessErrorMatching(
238
235
  "definitionInvalid",
239
236
  'hook: kind must be one of afterTool, beforeModelRequest, beforeTool, not "onEverything"'
@@ -774,7 +771,8 @@ describe("Extensions, hooks, wraps, and memos", () => {
774
771
  ...tool,
775
772
  execute: async (args, api) => {
776
773
  ran.push(`${tag}:before`);
777
- return {[tag]: await tool.execute(args, api)};
774
+ // `args` was parsed by the wrapped tool's own `parameters`.
775
+ return {[tag]: await tool.execute(args as never, api)};
778
776
  },
779
777
  });
780
778
  // The wrapping extension comes before the overriding one, yet wraps its tool.
@@ -1053,7 +1051,7 @@ describe("Extensions, hooks, wraps, and memos", () => {
1053
1051
  version: 1,
1054
1052
  });
1055
1053
  const harness = await openHarness({registry: [task]});
1056
- const ids = [];
1054
+ const ids: mongoose.Types.ObjectId[] = [];
1057
1055
  for (let n = 0; n < count; n++) {
1058
1056
  ids.push((await harness.createTask(task, {n}))._id);
1059
1057
  }
@@ -1106,7 +1104,7 @@ describe("Extensions, hooks, wraps, and memos", () => {
1106
1104
  it("lets exactly one of two independent runs win a key (first write wins)", async () => {
1107
1105
  const {release, rows} = await hangTasks(2);
1108
1106
  try {
1109
- const [first, second] = rows as [HarnessTaskDocument, HarnessTaskDocument];
1107
+ const [first, second] = rows;
1110
1108
  // Two runs fencing different task rows race on one shared key, so neither queues
1111
1109
  // behind the other's lease write and the unique index decides.
1112
1110
  const scope = first._id;
@@ -1128,7 +1126,7 @@ describe("Extensions, hooks, wraps, and memos", () => {
1128
1126
  it("refuses a write from a run whose lease token is stale", async () => {
1129
1127
  const {release, rows} = await hangTasks(1);
1130
1128
  try {
1131
- const [row] = rows as [HarnessTaskDocument];
1129
+ const [row] = rows;
1132
1130
  const stale = memoFor(row, row._id, "stale-token")("choice", "a");
1133
1131
  await expect(stale).rejects.toBeInstanceOf(HarnessCommitConflictError);
1134
1132
  await expect(stale).rejects.toThrow(
@@ -5,7 +5,12 @@ import {createLocalObservabilityPlugin} from "../observability/local/localPlugin
5
5
  import {registerObsSpan} from "../observability/local/models/obsSpan";
6
6
  import {registerObsTrace} from "../observability/local/models/obsTrace";
7
7
  import {harnessErrorMatching} from "../tests/harnessErrors";
8
- import type {HarnessChildOutcome, HarnessTaskDocument, HarnessTestHooks} from "../types/harness";
8
+ import type {
9
+ HarnessChildOutcome,
10
+ HarnessTaskDefinition,
11
+ HarnessTaskDocument,
12
+ HarnessTestHooks,
13
+ } from "../types/harness";
9
14
  import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
10
15
  import type {HarnessModels} from "./commit";
11
16
  import {
@@ -20,6 +25,9 @@ import {registerHarnessOwner} from "./models/harnessOwner";
20
25
  import {registerHarnessTask} from "./models/harnessTask";
21
26
  import {type HarnessEngine, sweepWaitingTasks} from "./ownership";
22
27
 
28
+ /** A task definition whose input is not checked, so tests can create it with any input. */
29
+ type TestTaskDefinition = HarnessTaskDefinition<unknown, unknown, unknown>;
30
+
23
31
  // Multi-task trees under a loaded full run need more than Bun's 5s default.
24
32
  setDefaultTimeout(30_000);
25
33
 
@@ -61,10 +69,9 @@ const findTask = (id: mongoose.Types.ObjectId | string): Promise<HarnessTaskDocu
61
69
  const childrenOf = (id: mongoose.Types.ObjectId): Promise<HarnessTaskDocument[]> =>
62
70
  TaskModel.find({"ownership.id": id, "ownership.kind": "task"}).sort({created: 1});
63
71
 
64
- const asAny = (definition: unknown): AnyHarnessTaskDefinition =>
65
- definition as AnyHarnessTaskDefinition;
72
+ const asAny = (definition: unknown): TestTaskDefinition => definition as TestTaskDefinition;
66
73
 
67
- const completesWith = (name: string, result: unknown): AnyHarnessTaskDefinition =>
74
+ const completesWith = (name: string, result: unknown): TestTaskDefinition =>
68
75
  asAny(
69
76
  defineTask<unknown, unknown, unknown>({
70
77
  initial: () => ({phase: "work"}),
@@ -76,7 +83,7 @@ const completesWith = (name: string, result: unknown): AnyHarnessTaskDefinition
76
83
  })
77
84
  );
78
85
 
79
- const failsWith = (name: string, error: string): AnyHarnessTaskDefinition =>
86
+ const failsWith = (name: string, error: string): TestTaskDefinition =>
80
87
  asAny(
81
88
  defineTask<unknown, unknown, unknown>({
82
89
  initial: () => ({phase: "work"}),
@@ -103,11 +110,11 @@ const fanOut = ({
103
110
  policy,
104
111
  runs,
105
112
  }: {
106
- children: AnyHarnessTaskDefinition[];
113
+ children: TestTaskDefinition[];
107
114
  name: string;
108
115
  policy?: "all" | "failFast";
109
116
  runs: {count: number};
110
- }): AnyHarnessTaskDefinition =>
117
+ }): TestTaskDefinition =>
111
118
  asAny(
112
119
  defineTask<unknown, unknown, HarnessChildOutcome[]>({
113
120
  initial: () => ({phase: "fanOut"}),
@@ -790,7 +797,9 @@ describe("Harness ownership tree", () => {
790
797
  expect(second.status).toBe("aborted");
791
798
  expect(await SpanModel.countDocuments({name: "abort", traceId: created.traceId})).toBe(1);
792
799
  // The first request recorded wins; the other waited for its handler to finish.
793
- expect(["first", "second"]).toContain((await findTask(created._id)).abortRequested?.reason);
800
+ expect<Array<string | undefined>>(["first", "second"]).toContain(
801
+ (await findTask(created._id)).abortRequested?.reason
802
+ );
794
803
  });
795
804
 
796
805
  it("keeps aborting the tree when a child's handler throws", async () => {
@@ -940,7 +949,7 @@ describe("Harness ownership tree", () => {
940
949
 
941
950
  describe("waiting sweep", () => {
942
951
  const buildEngine = (
943
- definitions: ReadonlyArray<AnyHarnessTaskDefinition> = []
952
+ definitions: ReadonlyArray<TestTaskDefinition> = []
944
953
  ): {engine: HarnessEngine; wakes: {count: number}} => {
945
954
  const wakes = {count: 0};
946
955
  const models = {
@@ -953,8 +962,10 @@ describe("Harness ownership tree", () => {
953
962
  } as unknown as HarnessModels;
954
963
  return {
955
964
  engine: {
965
+ agents: new Map(),
956
966
  controllers: new Map(),
957
967
  definitions: new Map(definitions.map((definition) => [definition.key, definition])),
968
+ extensions: new Map(),
958
969
  models,
959
970
  wake: () => {
960
971
  wakes.count += 1;