@terreno/ai 58.0.0-beta.2 → 58.0.0-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. package/CHANGELOG.md +146 -0
  2. package/README.md +3 -3
  3. package/dist/harness/commit.d.ts +4 -3
  4. package/dist/harness/commit.js +23 -4
  5. package/dist/index.d.ts +6 -9
  6. package/dist/index.js +14 -25
  7. package/dist/observability/adminScreens.js +5 -1
  8. package/dist/observability/evaluate.d.ts +8 -1
  9. package/dist/observability/evaluate.js +10 -2
  10. package/dist/observability/judgeAgreement.d.ts +53 -0
  11. package/dist/observability/judgeAgreement.js +123 -0
  12. package/dist/observability/local/datasetStore.d.ts +5 -0
  13. package/dist/observability/local/datasetStore.js +7 -2
  14. package/dist/observability/local/evaluatorStore.d.ts +17 -0
  15. package/dist/observability/local/evaluatorStore.js +73 -2
  16. package/dist/observability/local/experimentRunner.d.ts +40 -3
  17. package/dist/observability/local/experimentRunner.js +390 -11
  18. package/dist/observability/local/humanAnswers.d.ts +57 -0
  19. package/dist/observability/local/humanAnswers.js +174 -0
  20. package/dist/observability/local/judgeRun.d.ts +56 -0
  21. package/dist/observability/local/judgeRun.js +111 -0
  22. package/dist/observability/local/localPlugin.js +4 -0
  23. package/dist/observability/local/models/obsDatasetItem.js +4 -0
  24. package/dist/observability/local/models/obsEvaluator.js +4 -0
  25. package/dist/observability/local/models/obsExperiment.js +28 -0
  26. package/dist/observability/local/models/obsExperimentItem.js +4 -0
  27. package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
  28. package/dist/observability/local/models/obsJudgeRun.js +60 -0
  29. package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
  30. package/dist/observability/local/models/obsReviewBatch.js +48 -0
  31. package/dist/observability/local/models/obsReviewItem.js +10 -2
  32. package/dist/observability/local/models/obsTrace.js +4 -0
  33. package/dist/observability/local/promptReferences.d.ts +31 -0
  34. package/dist/observability/local/promptReferences.js +114 -0
  35. package/dist/observability/local/promptStore.d.ts +55 -19
  36. package/dist/observability/local/promptStore.js +171 -7
  37. package/dist/observability/local/reviewBatchStore.d.ts +106 -0
  38. package/dist/observability/local/reviewBatchStore.js +517 -0
  39. package/dist/observability/local/reviewStore.d.ts +24 -3
  40. package/dist/observability/local/reviewStore.js +98 -22
  41. package/dist/observability/local/traceStore.d.ts +14 -1
  42. package/dist/observability/local/traceStore.js +144 -11
  43. package/dist/observability/observabilityApp.js +10 -1
  44. package/dist/observability/questionFingerprint.d.ts +7 -0
  45. package/dist/observability/questionFingerprint.js +23 -0
  46. package/dist/observability/routes/experiments.js +12 -1
  47. package/dist/observability/routes/review.js +3 -1
  48. package/dist/observability/routes/reviewBatches.d.ts +11 -0
  49. package/dist/observability/routes/reviewBatches.js +78 -0
  50. package/dist/observability/routes/traces.d.ts +1 -1
  51. package/dist/observability/routes/traces.js +34 -2
  52. package/dist/observability/types.d.ts +10 -1
  53. package/dist/service/aiService.js +4 -1
  54. package/dist/service/chatTurn.js +38 -31
  55. package/dist/types/harness.d.ts +5 -2
  56. package/dist/types/index.d.ts +8 -3
  57. package/dist/types/observability.d.ts +74 -2
  58. package/package.json +8 -24
  59. package/src/harness/agentLoop.test.ts +0 -3
  60. package/src/harness/agentRecovery.test.ts +0 -3
  61. package/src/harness/commit.ts +24 -4
  62. package/src/harness/extensions.test.ts +0 -3
  63. package/src/harness/subagent.test.ts +27 -3
  64. package/src/index.test.ts +0 -17
  65. package/src/index.ts +23 -26
  66. package/src/isolated/gptStream.isolated.ts +0 -8
  67. package/src/observability/adminScreens.test.ts +7 -0
  68. package/src/observability/adminScreens.ts +6 -1
  69. package/src/observability/evaluate.ts +12 -2
  70. package/src/observability/judgeAgreement.test.ts +111 -0
  71. package/src/observability/judgeAgreement.ts +184 -0
  72. package/src/observability/local/datasetStore.ts +30 -11
  73. package/src/observability/local/evaluatorStore.ts +96 -2
  74. package/src/observability/local/experimentRunner.ts +495 -15
  75. package/src/observability/local/humanAnswers.ts +233 -0
  76. package/src/observability/local/judgePipeline.test.ts +444 -0
  77. package/src/observability/local/judgeRun.test.ts +173 -0
  78. package/src/observability/local/judgeRun.ts +184 -0
  79. package/src/observability/local/localPlugin.ts +4 -0
  80. package/src/observability/local/models/obsDatasetItem.ts +5 -0
  81. package/src/observability/local/models/obsEvaluator.ts +4 -0
  82. package/src/observability/local/models/obsExperiment.ts +29 -0
  83. package/src/observability/local/models/obsExperimentItem.ts +4 -0
  84. package/src/observability/local/models/obsJudgeRun.ts +61 -0
  85. package/src/observability/local/models/obsReviewBatch.ts +52 -0
  86. package/src/observability/local/models/obsReviewItem.ts +11 -2
  87. package/src/observability/local/models/obsTrace.ts +5 -0
  88. package/src/observability/local/obsModels.test.ts +13 -0
  89. package/src/observability/local/promptReferences.test.ts +128 -0
  90. package/src/observability/local/promptReferences.ts +149 -0
  91. package/src/observability/local/promptStore.test.ts +291 -0
  92. package/src/observability/local/promptStore.ts +243 -32
  93. package/src/observability/local/reviewBatchStore.test.ts +251 -0
  94. package/src/observability/local/reviewBatchStore.ts +656 -0
  95. package/src/observability/local/reviewStore.ts +122 -25
  96. package/src/observability/local/traceStore.test.ts +139 -1
  97. package/src/observability/local/traceStore.ts +162 -11
  98. package/src/observability/observabilityApp.test.ts +13 -10
  99. package/src/observability/observabilityApp.ts +10 -1
  100. package/src/observability/questionFingerprint.test.ts +33 -0
  101. package/src/observability/questionFingerprint.ts +24 -0
  102. package/src/observability/routes/experiments.test.ts +37 -0
  103. package/src/observability/routes/experiments.ts +13 -1
  104. package/src/observability/routes/review.ts +3 -1
  105. package/src/observability/routes/reviewBatches.test.ts +209 -0
  106. package/src/observability/routes/reviewBatches.ts +138 -0
  107. package/src/observability/routes/status.test.ts +4 -4
  108. package/src/observability/routes/traces.test.ts +61 -0
  109. package/src/observability/routes/traces.ts +46 -4
  110. package/src/observability/types.ts +5 -2
  111. package/src/routes/routes.test.ts +101 -76
  112. package/src/service/aiService.test.ts +21 -0
  113. package/src/service/aiService.ts +4 -1
  114. package/src/service/chatTurn.ts +42 -33
  115. package/src/tests/bunSetup.ts +0 -54
  116. package/src/types/harness.ts +5 -2
  117. package/src/types/index.ts +8 -3
  118. package/src/types/observability.ts +86 -2
  119. package/dist/langfuseApp.d.ts +0 -9
  120. package/dist/langfuseApp.js +0 -53
  121. package/dist/langfuseCache.d.ts +0 -15
  122. package/dist/langfuseCache.js +0 -54
  123. package/dist/langfuseClient.d.ts +0 -6
  124. package/dist/langfuseClient.js +0 -34
  125. package/dist/langfusePrompts.d.ts +0 -12
  126. package/dist/langfusePrompts.js +0 -89
  127. package/dist/langfuseRoutesEvaluations.d.ts +0 -3
  128. package/dist/langfuseRoutesEvaluations.js +0 -32
  129. package/dist/langfuseRoutesMiddleware.d.ts +0 -2
  130. package/dist/langfuseRoutesMiddleware.js +0 -13
  131. package/dist/langfuseRoutesPlayground.d.ts +0 -2
  132. package/dist/langfuseRoutesPlayground.js +0 -36
  133. package/dist/langfuseRoutesPrompts.d.ts +0 -2
  134. package/dist/langfuseRoutesPrompts.js +0 -56
  135. package/dist/langfuseRoutesTraces.d.ts +0 -2
  136. package/dist/langfuseRoutesTraces.js +0 -30
  137. package/dist/langfuseTracing.d.ts +0 -4
  138. package/dist/langfuseTracing.js +0 -32
  139. package/dist/langfuseTypes.d.ts +0 -96
  140. package/dist/langfuseTypes.js +0 -2
  141. package/dist/langfuseVercelAi.d.ts +0 -12
  142. package/dist/langfuseVercelAi.js +0 -49
  143. package/src/isolated/langfuseClient.isolated.ts +0 -66
  144. package/src/langfuseApp.test.ts +0 -119
  145. package/src/langfuseApp.ts +0 -62
  146. package/src/langfuseCache.test.ts +0 -75
  147. package/src/langfuseCache.ts +0 -72
  148. package/src/langfuseClient.test.ts +0 -69
  149. package/src/langfuseClient.ts +0 -34
  150. package/src/langfusePrompts.test.ts +0 -188
  151. package/src/langfusePrompts.ts +0 -121
  152. package/src/langfuseRoutes.test.ts +0 -261
  153. package/src/langfuseRoutesEvaluations.ts +0 -48
  154. package/src/langfuseRoutesMiddleware.test.ts +0 -57
  155. package/src/langfuseRoutesMiddleware.ts +0 -12
  156. package/src/langfuseRoutesPlayground.ts +0 -50
  157. package/src/langfuseRoutesPrompts.ts +0 -87
  158. package/src/langfuseRoutesTraces.ts +0 -40
  159. package/src/langfuseTracing.test.ts +0 -72
  160. package/src/langfuseTracing.ts +0 -33
  161. package/src/langfuseTypes.ts +0 -103
  162. package/src/langfuseVercelAi.test.ts +0 -82
  163. package/src/langfuseVercelAi.ts +0 -72
  164. package/src/tests/fakeLangfusePrompt.ts +0 -10
@@ -1,6 +1,8 @@
1
1
  import type {FindExactlyOnePlugin, FindOneOrNonePlugin} from "@terreno/api";
2
2
  import type mongoose from "mongoose";
3
3
 
4
+ import type {JudgeAgreement} from "../observability/judgeAgreement";
5
+
4
6
  export interface ObsPromptVariable {
5
7
  key: string;
6
8
  label?: string;
@@ -16,6 +18,8 @@ export interface ObsPromptUsage {
16
18
  }
17
19
 
18
20
  export interface ObsPromptRef {
21
+ /** Set when this version was pulled in by another prompt's reference, to that prompt's name. */
22
+ includedBy?: string;
19
23
  label?: string;
20
24
  name: string;
21
25
  version: number;
@@ -157,6 +161,8 @@ export interface ObsScoreModel extends mongoose.Model<ObsScoreDocument>, ObsScor
157
161
  export interface EvaluatorDimension {
158
162
  dataType: "boolean" | "categorical" | "numeric";
159
163
  key: string;
164
+ /** Question shown to human reviewers, e.g. "Does the summary mention every todo?" */
165
+ question?: string;
160
166
  range?: string;
161
167
  required: boolean;
162
168
  }
@@ -193,6 +199,7 @@ export interface ObsEvaluatorModel
193
199
 
194
200
  export interface ObsReviewItemDocument extends mongoose.Document<mongoose.Types.ObjectId> {
195
201
  assigneeId?: mongoose.Types.ObjectId;
202
+ batchId?: mongoose.Types.ObjectId;
196
203
  comment?: string;
197
204
  created: Date;
198
205
  datasetItemId?: mongoose.Types.ObjectId;
@@ -203,7 +210,8 @@ export interface ObsReviewItemDocument extends mongoose.Document<mongoose.Types.
203
210
  scores?: Record<string, boolean | number | string>;
204
211
  spanId?: mongoose.Types.ObjectId;
205
212
  status: "done" | "in_progress" | "pending" | "skipped";
206
- traceId: mongoose.Types.ObjectId;
213
+ /** Trace under review. Unset when reviewers judge a dataset item that has no source trace. */
214
+ traceId?: mongoose.Types.ObjectId;
207
215
  updated: Date;
208
216
  }
209
217
 
@@ -215,6 +223,54 @@ export interface ObsReviewItemModel
215
223
  extends mongoose.Model<ObsReviewItemDocument>,
216
224
  ObsReviewItemStatics {}
217
225
 
226
+ export interface ObsReviewBatchDocument extends mongoose.Document<mongoose.Types.ObjectId> {
227
+ created: Date;
228
+ /** Dataset the batch was built from, when reviewers judge dataset items. */
229
+ datasetId?: mongoose.Types.ObjectId;
230
+ deleted: boolean;
231
+ description?: string;
232
+ evaluatorId: mongoose.Types.ObjectId;
233
+ name: string;
234
+ promptName?: string;
235
+ /** Question-set fingerprint when the batch was created (see `questionSetFingerprint`). */
236
+ questionsFingerprint?: string;
237
+ traceIds: mongoose.Types.ObjectId[];
238
+ updated: Date;
239
+ }
240
+
241
+ export interface ObsReviewBatchStatics
242
+ extends FindExactlyOnePlugin<ObsReviewBatchDocument>,
243
+ FindOneOrNonePlugin<ObsReviewBatchDocument> {}
244
+
245
+ export interface ObsReviewBatchModel
246
+ extends mongoose.Model<ObsReviewBatchDocument>,
247
+ ObsReviewBatchStatics {}
248
+
249
+ export interface ObsJudgeRunResult {
250
+ confidence?: number;
251
+ datasetItemId?: mongoose.Types.ObjectId;
252
+ error?: string;
253
+ scores?: Record<string, boolean | number | string>;
254
+ traceId?: mongoose.Types.ObjectId;
255
+ }
256
+
257
+ export interface ObsJudgeRunDocument extends mongoose.Document<mongoose.Types.ObjectId> {
258
+ batchId: mongoose.Types.ObjectId;
259
+ completedAt: Date;
260
+ created: Date;
261
+ deleted: boolean;
262
+ evaluatorId: mongoose.Types.ObjectId;
263
+ modelId?: string;
264
+ results: ObsJudgeRunResult[];
265
+ updated: Date;
266
+ }
267
+
268
+ export interface ObsJudgeRunStatics
269
+ extends FindExactlyOnePlugin<ObsJudgeRunDocument>,
270
+ FindOneOrNonePlugin<ObsJudgeRunDocument> {}
271
+
272
+ export interface ObsJudgeRunModel extends mongoose.Model<ObsJudgeRunDocument>, ObsJudgeRunStatics {}
273
+
218
274
  export interface ObsDatasetAnnotatedBy {
219
275
  label: string;
220
276
  reviewItemId?: string;
@@ -238,12 +294,24 @@ export interface ObsDatasetStatics
238
294
 
239
295
  export interface ObsDatasetModel extends mongoose.Model<ObsDatasetDocument>, ObsDatasetStatics {}
240
296
 
297
+ /** One question set's human answers for a dataset item. */
298
+ export interface ObsDatasetHumanAnswer {
299
+ answeredAt: Date;
300
+ /** `questionSetFingerprint` of the question set when the reviewer answered. */
301
+ fingerprint: string;
302
+ reviewItemId: string;
303
+ scores: Record<string, boolean | number | string>;
304
+ userId?: string;
305
+ }
306
+
241
307
  export interface ObsDatasetItemDocument extends mongoose.Document<mongoose.Types.ObjectId> {
242
308
  annotatedBy?: ObsDatasetAnnotatedBy;
243
309
  created: Date;
244
310
  datasetId: mongoose.Types.ObjectId;
245
311
  deleted: boolean;
246
312
  expectedOutput?: unknown;
313
+ /** Human answers keyed by question-set (human evaluator) id. */
314
+ humanAnswers?: Record<string, ObsDatasetHumanAnswer>;
247
315
  input: unknown;
248
316
  metadata?: Record<string, unknown>;
249
317
  origin: "manual" | "synthetic" | "trace";
@@ -263,7 +331,8 @@ export interface ObsDatasetItemModel
263
331
  ObsDatasetItemStatics {}
264
332
 
265
333
  export interface ScoreThreshold {
266
- aggregate: "mean" | "trueRate";
334
+ /** `agreement` gates compare a judge prompt version's answers with human answers. */
335
+ aggregate: "agreement" | "mean" | "trueRate";
267
336
  dimension: string;
268
337
  evaluatorName: string;
269
338
  op: "eq" | "gte" | "lte";
@@ -282,7 +351,11 @@ export interface ExperimentEstimate {
282
351
  wallClockSeconds: number;
283
352
  }
284
353
 
354
+ export type ExperimentKind = "judge" | "prompt";
355
+
285
356
  export interface ExperimentAggregates {
357
+ /** Judge experiments: agreement with the human answers per judge prompt version. */
358
+ agreement?: Record<string, JudgeAgreement>;
286
359
  gates: ExperimentGateResult[];
287
360
  lowConfidenceItemIds: string[];
288
361
  outlierItemIds: string[];
@@ -298,11 +371,20 @@ export interface ObsExperimentDocument extends mongoose.Document<mongoose.Types.
298
371
  estimate?: ExperimentEstimate;
299
372
  evaluatorIds: mongoose.Types.ObjectId[];
300
373
  includeUnproofread: boolean;
374
+ /** Judge experiments: the llm-judge evaluator whose prompt versions are compared. */
375
+ judgeEvaluatorId?: mongoose.Types.ObjectId;
376
+ kind: ExperimentKind;
301
377
  modelOverride?: string;
302
378
  name: string;
303
379
  promptName: string;
380
+ /** Judge experiments: question-set fingerprint of the human answers compared. */
381
+ questionsFingerprint?: string;
382
+ /** Judge experiments: the human question set whose answers are the reference. */
383
+ questionSetId?: mongoose.Types.ObjectId;
304
384
  results?: ExperimentAggregates;
305
385
  status: "completed" | "failed" | "pending" | "running";
386
+ /** Judge experiments: agreement rate (0–1) a version must reach to pass. */
387
+ targetAgreement?: number;
306
388
  thresholds: ScoreThreshold[];
307
389
  updated: Date;
308
390
  versions: number[];
@@ -334,6 +416,8 @@ export interface ObsExperimentItemDocument extends mongoose.Document<mongoose.Ty
334
416
  deleted: boolean;
335
417
  experimentId: mongoose.Types.ObjectId;
336
418
  failed: boolean;
419
+ /** Judge experiments: the human answers the judge versions are compared with. */
420
+ humanScores?: Record<string, boolean | number | string>;
337
421
  updated: Date;
338
422
  versionResults: Record<string, ExperimentVersionResult>;
339
423
  }
@@ -1,9 +0,0 @@
1
- import type { TerrenoPlugin } from "@terreno/api";
2
- import type express from "express";
3
- import type { LangfuseAppOptions } from "./langfuseTypes";
4
- export declare class LangfuseApp implements TerrenoPlugin {
5
- private options;
6
- constructor(options: LangfuseAppOptions);
7
- register(app: express.Application): void;
8
- shutdown(): Promise<void>;
9
- }
@@ -1,53 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.LangfuseApp = void 0;
4
- const api_1 = require("@terreno/api");
5
- const langfuseClient_1 = require("./langfuseClient");
6
- const langfuseRoutesEvaluations_1 = require("./langfuseRoutesEvaluations");
7
- const langfuseRoutesPlayground_1 = require("./langfuseRoutesPlayground");
8
- const langfuseRoutesPrompts_1 = require("./langfuseRoutesPrompts");
9
- const langfuseRoutesTraces_1 = require("./langfuseRoutesTraces");
10
- const langfuseTracing_1 = require("./langfuseTracing");
11
- class LangfuseApp {
12
- options;
13
- constructor(options) {
14
- this.options = options;
15
- }
16
- register(app) {
17
- const organization = this.options.organization ?? "flourish-health";
18
- const project = this.options.project ?? "terreno";
19
- try {
20
- (0, langfuseClient_1.initLangfuseClient)(this.options);
21
- }
22
- catch (err) {
23
- api_1.logger.warn(`Langfuse client initialization failed; continuing without Langfuse: ${err}`);
24
- return;
25
- }
26
- if (this.options.enableTracing !== false) {
27
- try {
28
- (0, langfuseTracing_1.initTracing)(this.options);
29
- }
30
- catch (err) {
31
- api_1.logger.warn(`Langfuse tracing initialization failed: ${err}`);
32
- }
33
- }
34
- if (this.options.enableAdminUI !== false) {
35
- const adminPath = this.options.adminPath ?? "/admin/langfuse";
36
- (0, langfuseRoutesPrompts_1.addPromptRoutes)(app, adminPath);
37
- (0, langfuseRoutesTraces_1.addTraceRoutes)(app, adminPath);
38
- (0, langfuseRoutesPlayground_1.addPlaygroundRoutes)(app, adminPath);
39
- if (this.options.evaluation?.enabled) {
40
- (0, langfuseRoutesEvaluations_1.addEvaluationRoutes)(app, adminPath, this.options.evaluation.scoringFunctions ?? []);
41
- }
42
- api_1.logger.info(`Langfuse admin routes mounted at ${adminPath} (org: ${organization}, project: ${project})`);
43
- }
44
- process.once("SIGTERM", () => {
45
- void this.shutdown();
46
- });
47
- }
48
- async shutdown() {
49
- await (0, langfuseClient_1.shutdownLangfuseClient)();
50
- await (0, langfuseTracing_1.shutdownTracing)();
51
- }
52
- }
53
- exports.LangfuseApp = LangfuseApp;
@@ -1,15 +0,0 @@
1
- import mongoose, { type Document } from "mongoose";
2
- import type { LangfuseCachedPrompt } from "./langfuseTypes";
3
- interface LangfuseCacheDocument extends mongoose.Document {
4
- key: string;
5
- value: LangfuseCachedPrompt;
6
- expiresAt: Date;
7
- }
8
- interface LangfuseCacheModel extends mongoose.Model<LangfuseCacheDocument> {
9
- findOneOrNone(query: Record<string, unknown>): Promise<(Document & LangfuseCacheDocument) | null>;
10
- }
11
- export declare const LangfuseCache: LangfuseCacheModel;
12
- export declare const getCached: (key: string) => Promise<LangfuseCachedPrompt | null>;
13
- export declare const setCached: (key: string, value: LangfuseCachedPrompt, ttlSeconds: number) => Promise<void>;
14
- export declare const invalidateCache: (keyPattern: string) => Promise<void>;
15
- export {};
@@ -1,54 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.invalidateCache = exports.setCached = exports.getCached = exports.LangfuseCache = void 0;
7
- const api_1 = require("@terreno/api");
8
- const luxon_1 = require("luxon");
9
- const mongoose_1 = __importDefault(require("mongoose"));
10
- const langfuseCacheSchema = new mongoose_1.default.Schema({
11
- expiresAt: {
12
- description: "When this cache entry expires",
13
- required: true,
14
- type: Date,
15
- },
16
- key: {
17
- description: "Cache key (e.g. prompt:chat-assistant:production)",
18
- required: true,
19
- type: String,
20
- unique: true,
21
- },
22
- value: {
23
- description: "Cached Langfuse object",
24
- required: true,
25
- type: mongoose_1.default.Schema.Types.Mixed,
26
- },
27
- }, { strict: "throw", toJSON: { virtuals: true }, toObject: { virtuals: true } });
28
- langfuseCacheSchema.index({ expiresAt: 1 }, { expireAfterSeconds: 0 });
29
- langfuseCacheSchema.plugin(api_1.createdUpdatedPlugin);
30
- langfuseCacheSchema.plugin(api_1.isDeletedPlugin);
31
- langfuseCacheSchema.plugin(api_1.findOneOrNone);
32
- langfuseCacheSchema.plugin(api_1.findExactlyOne);
33
- exports.LangfuseCache = mongoose_1.default.models.LangfuseCache ||
34
- mongoose_1.default.model("LangfuseCache", langfuseCacheSchema);
35
- const getCached = async (key) => {
36
- const entry = await exports.LangfuseCache.findOneOrNone({
37
- expiresAt: { $gt: luxon_1.DateTime.now().toJSDate() },
38
- key,
39
- });
40
- if (!entry) {
41
- return null;
42
- }
43
- return entry.value;
44
- };
45
- exports.getCached = getCached;
46
- const setCached = async (key, value, ttlSeconds) => {
47
- const expiresAt = luxon_1.DateTime.now().plus({ seconds: ttlSeconds }).toJSDate();
48
- await exports.LangfuseCache.findOneAndUpdate({ key }, { expiresAt, key, value }, { upsert: true });
49
- };
50
- exports.setCached = setCached;
51
- const invalidateCache = async (keyPattern) => {
52
- await exports.LangfuseCache.deleteMany({ key: { $regex: keyPattern } });
53
- };
54
- exports.invalidateCache = invalidateCache;
@@ -1,6 +0,0 @@
1
- import { LangfuseClient } from "@langfuse/client";
2
- import type { LangfuseAppOptions } from "./langfuseTypes";
3
- export declare const initLangfuseClient: (options: LangfuseAppOptions) => LangfuseClient;
4
- export declare const getLangfuseClient: () => LangfuseClient;
5
- export declare const isLangfuseInitialized: () => boolean;
6
- export declare const shutdownLangfuseClient: () => Promise<void>;
@@ -1,34 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.shutdownLangfuseClient = exports.isLangfuseInitialized = exports.getLangfuseClient = exports.initLangfuseClient = void 0;
4
- const client_1 = require("@langfuse/client");
5
- const api_1 = require("@terreno/api");
6
- let langfuseInstance = null;
7
- const initLangfuseClient = (options) => {
8
- langfuseInstance = new client_1.LangfuseClient({
9
- baseUrl: options.baseUrl ?? "https://cloud.langfuse.com",
10
- publicKey: options.publicKey,
11
- secretKey: options.secretKey,
12
- });
13
- return langfuseInstance;
14
- };
15
- exports.initLangfuseClient = initLangfuseClient;
16
- const getLangfuseClient = () => {
17
- if (!langfuseInstance) {
18
- throw new api_1.APIError({
19
- status: 500,
20
- title: "Langfuse client not initialized. Call initLangfuseClient first.",
21
- });
22
- }
23
- return langfuseInstance;
24
- };
25
- exports.getLangfuseClient = getLangfuseClient;
26
- const isLangfuseInitialized = () => langfuseInstance != null;
27
- exports.isLangfuseInitialized = isLangfuseInitialized;
28
- const shutdownLangfuseClient = async () => {
29
- if (langfuseInstance) {
30
- await langfuseInstance.shutdown();
31
- langfuseInstance = null;
32
- }
33
- };
34
- exports.shutdownLangfuseClient = shutdownLangfuseClient;
@@ -1,12 +0,0 @@
1
- import type { ChatMessage, GetPromptOptions, LangfuseAppOptions, LangfuseCachedPrompt } from "./langfuseTypes";
2
- export declare const getPrompt: (name: string, options?: GetPromptOptions, appOptions?: Pick<LangfuseAppOptions, "cache">) => Promise<LangfuseCachedPrompt>;
3
- export declare const compilePrompt: (cached: LangfuseCachedPrompt, variables?: Record<string, string>) => string | ChatMessage[];
4
- export declare const createPrompt: (params: {
5
- name: string;
6
- type: "text" | "chat";
7
- prompt: string | ChatMessage[];
8
- labels?: string[];
9
- tags?: string[];
10
- config?: Record<string, unknown>;
11
- }) => Promise<LangfuseCachedPrompt>;
12
- export declare const invalidatePromptCache: (name: string) => Promise<void>;
@@ -1,89 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.invalidatePromptCache = exports.createPrompt = exports.compilePrompt = exports.getPrompt = void 0;
4
- const api_1 = require("@terreno/api");
5
- const langfuseCache_1 = require("./langfuseCache");
6
- const langfuseClient_1 = require("./langfuseClient");
7
- const DEFAULT_PROMPT_TTL = 60;
8
- const DEFAULT_LABEL = "production";
9
- const buildPromptCacheKey = (name, label) => {
10
- return `prompt:${name}:${label ?? DEFAULT_LABEL}`;
11
- };
12
- const getPrompt = async (name, options = {}, appOptions) => {
13
- const ttl = appOptions?.cache?.promptTtlSeconds ?? DEFAULT_PROMPT_TTL;
14
- const cacheKey = buildPromptCacheKey(name, options.label);
15
- const cached = await (0, langfuseCache_1.getCached)(cacheKey);
16
- if (cached) {
17
- api_1.logger.debug(`Langfuse prompt cache hit: "${name}" v${cached.version} (label: ${options.label ?? DEFAULT_LABEL})`);
18
- return cached;
19
- }
20
- api_1.logger.info(`Langfuse prompt cache miss: fetching "${name}" from API (label: ${options.label ?? DEFAULT_LABEL})`);
21
- const client = (0, langfuseClient_1.getLangfuseClient)();
22
- const prompt = await client.prompt.get(name, {
23
- cacheTtlSeconds: 0,
24
- label: options.label ?? DEFAULT_LABEL,
25
- });
26
- const cachedPrompt = {
27
- config: (prompt.config ?? {}),
28
- labels: prompt.labels ?? [],
29
- name: prompt.name,
30
- prompt: prompt.prompt,
31
- tags: prompt.tags ?? [],
32
- type: prompt.type,
33
- version: prompt.version,
34
- };
35
- api_1.logger.info(`Langfuse prompt fetched: "${name}" v${cachedPrompt.version} (ttl: ${ttl}s)`);
36
- await (0, langfuseCache_1.setCached)(cacheKey, cachedPrompt, ttl);
37
- return cachedPrompt;
38
- };
39
- exports.getPrompt = getPrompt;
40
- const compilePrompt = (cached, variables = {}) => {
41
- const replace = (template) => {
42
- return template.replace(/\{\{(\w+)\}\}/g, (_match, key) => {
43
- return variables[key] ?? `{{${key}}}`;
44
- });
45
- };
46
- if (cached.type === "text") {
47
- return replace(cached.prompt);
48
- }
49
- return cached.prompt.map((msg) => ({
50
- content: replace(msg.content),
51
- role: msg.role,
52
- }));
53
- };
54
- exports.compilePrompt = compilePrompt;
55
- const createPrompt = async (params) => {
56
- const client = (0, langfuseClient_1.getLangfuseClient)();
57
- if (params.type === "text") {
58
- await client.prompt.create({
59
- config: params.config,
60
- labels: params.labels ?? [DEFAULT_LABEL],
61
- name: params.name,
62
- prompt: params.prompt,
63
- tags: params.tags ?? [],
64
- type: "text",
65
- });
66
- }
67
- else {
68
- const chatMessages = params.prompt.map((msg) => ({
69
- content: msg.content,
70
- role: msg.role,
71
- }));
72
- await client.prompt.create({
73
- config: params.config,
74
- labels: params.labels ?? [DEFAULT_LABEL],
75
- name: params.name,
76
- prompt: chatMessages,
77
- tags: params.tags ?? [],
78
- type: "chat",
79
- });
80
- }
81
- await (0, exports.invalidatePromptCache)(params.name);
82
- return (0, exports.getPrompt)(params.name, { label: DEFAULT_LABEL });
83
- };
84
- exports.createPrompt = createPrompt;
85
- const invalidatePromptCache = async (name) => {
86
- await (0, langfuseCache_1.invalidateCache)(`prompt:${name}:`);
87
- api_1.logger.info(`Langfuse prompt cache invalidated for: ${name}`);
88
- };
89
- exports.invalidatePromptCache = invalidatePromptCache;
@@ -1,3 +0,0 @@
1
- import type express from "express";
2
- import type { ScoringFunction } from "./langfuseTypes";
3
- export declare const addEvaluationRoutes: (router: express.Application, basePath: string, scoringFunctions?: ScoringFunction[]) => void;
@@ -1,32 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.addEvaluationRoutes = void 0;
4
- const api_1 = require("@terreno/api");
5
- const langfuseClient_1 = require("./langfuseClient");
6
- const langfuseRoutesMiddleware_1 = require("./langfuseRoutesMiddleware");
7
- const addEvaluationRoutes = (router, basePath, scoringFunctions = []) => {
8
- router.get(`${basePath}/evaluations/config`, [(0, api_1.authenticateMiddleware)(), langfuseRoutesMiddleware_1.requireAdmin], (0, api_1.asyncHandler)(async (_req, res) => {
9
- return res.json({ scoringFunctions });
10
- }));
11
- router.post(`${basePath}/evaluations`, [(0, api_1.authenticateMiddleware)(), langfuseRoutesMiddleware_1.requireAdmin], (0, api_1.asyncHandler)(async (req, res) => {
12
- const { traceId, name, value, comment, dataType } = req.body;
13
- if (!traceId || !name || value === undefined) {
14
- throw new api_1.APIError({
15
- detail: "traceId, name, and value are required",
16
- status: 400,
17
- title: "Invalid request",
18
- });
19
- }
20
- const client = (0, langfuseClient_1.getLangfuseClient)();
21
- client.score.create({
22
- comment: comment ?? undefined,
23
- dataType: dataType,
24
- name,
25
- traceId,
26
- value,
27
- });
28
- await client.flush();
29
- return res.status(201).json({ comment, dataType, name, traceId, value });
30
- }));
31
- };
32
- exports.addEvaluationRoutes = addEvaluationRoutes;
@@ -1,2 +0,0 @@
1
- import type { NextFunction, Request, Response } from "express";
2
- export declare const requireAdmin: (req: Request, _res: Response, next: NextFunction) => void;
@@ -1,13 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.requireAdmin = void 0;
4
- const api_1 = require("@terreno/api");
5
- const requireAdmin = (req, _res, next) => {
6
- const user = req.user;
7
- const isAdmin = typeof user?.isAdmin === "function" ? user.isAdmin() : api_1.Permissions.IsAdmin("read", user);
8
- if (!isAdmin) {
9
- throw new api_1.APIError({ disableExternalErrorTracking: true, status: 403, title: "Forbidden" });
10
- }
11
- next();
12
- };
13
- exports.requireAdmin = requireAdmin;
@@ -1,2 +0,0 @@
1
- import type express from "express";
2
- export declare const addPlaygroundRoutes: (router: express.Application, basePath: string) => void;
@@ -1,36 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.addPlaygroundRoutes = void 0;
4
- const api_1 = require("@terreno/api");
5
- const langfusePrompts_1 = require("./langfusePrompts");
6
- const langfuseRoutesMiddleware_1 = require("./langfuseRoutesMiddleware");
7
- const addPlaygroundRoutes = (router, basePath) => {
8
- router.post(`${basePath}/playground`, [(0, api_1.authenticateMiddleware)(), langfuseRoutesMiddleware_1.requireAdmin], (0, api_1.asyncHandler)(async (req, res) => {
9
- const { promptName, variables, label } = req.body;
10
- if (!promptName) {
11
- throw new api_1.APIError({
12
- detail: "promptName is required",
13
- status: 400,
14
- title: "Invalid request",
15
- });
16
- }
17
- const cached = await (0, langfusePrompts_1.getPrompt)(promptName, { label, variables });
18
- const compiled = (0, langfusePrompts_1.compilePrompt)(cached, variables ?? {});
19
- return res.json({
20
- compiled,
21
- config: cached.config,
22
- labels: cached.labels,
23
- name: cached.name,
24
- type: cached.type,
25
- variables: extractVariables(cached.type === "text"
26
- ? cached.prompt
27
- : cached.prompt.map((m) => m.content).join(" ")),
28
- version: cached.version,
29
- });
30
- }));
31
- };
32
- exports.addPlaygroundRoutes = addPlaygroundRoutes;
33
- const extractVariables = (template) => {
34
- const matches = template.match(/\{\{(\w+)\}\}/g) ?? [];
35
- return [...new Set(matches.map((m) => m.slice(2, -2)))];
36
- };
@@ -1,2 +0,0 @@
1
- import type express from "express";
2
- export declare const addPromptRoutes: (router: express.Application, basePath: string) => void;
@@ -1,56 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.addPromptRoutes = void 0;
4
- const api_1 = require("@terreno/api");
5
- const langfuseClient_1 = require("./langfuseClient");
6
- const langfusePrompts_1 = require("./langfusePrompts");
7
- const langfuseRoutesMiddleware_1 = require("./langfuseRoutesMiddleware");
8
- const addPromptRoutes = (router, basePath) => {
9
- router.get(`${basePath}/prompts`, [(0, api_1.authenticateMiddleware)(), langfuseRoutesMiddleware_1.requireAdmin], (0, api_1.asyncHandler)(async (req, res) => {
10
- const client = (0, langfuseClient_1.getLangfuseClient)();
11
- const page = Number(req.query.page) || 1;
12
- const limit = Number(req.query.limit) || 20;
13
- const result = await client.api.prompts.list({ limit, page });
14
- return res.json({
15
- data: result.data,
16
- meta: result.meta,
17
- });
18
- }));
19
- router.get(`${basePath}/prompts/:name`, [(0, api_1.authenticateMiddleware)(), langfuseRoutesMiddleware_1.requireAdmin], (0, api_1.asyncHandler)(async (req, res) => {
20
- const client = (0, langfuseClient_1.getLangfuseClient)();
21
- const { name } = req.params;
22
- const version = req.query.version ? Number(req.query.version) : undefined;
23
- const label = req.query.label;
24
- const prompt = await client.prompt.get(name, {
25
- cacheTtlSeconds: 0,
26
- ...(version !== undefined ? { version } : {}),
27
- ...(label ? { label } : {}),
28
- });
29
- return res.json({
30
- config: prompt.config,
31
- labels: prompt.labels,
32
- name: prompt.name,
33
- prompt: prompt.prompt,
34
- tags: prompt.tags,
35
- type: prompt.type,
36
- version: prompt.version,
37
- });
38
- }));
39
- router.post(`${basePath}/prompts`, [(0, api_1.authenticateMiddleware)(), langfuseRoutesMiddleware_1.requireAdmin], (0, api_1.asyncHandler)(async (req, res) => {
40
- const { name, type, prompt, labels, tags, config } = req.body;
41
- if (!name || !type || prompt === undefined) {
42
- throw new api_1.APIError({
43
- detail: "name, type, and prompt are required",
44
- status: 400,
45
- title: "Invalid request",
46
- });
47
- }
48
- const result = await (0, langfusePrompts_1.createPrompt)({ config, labels, name, prompt, tags, type });
49
- return res.status(201).json(result);
50
- }));
51
- router.delete(`${basePath}/prompts/:name/cache`, [(0, api_1.authenticateMiddleware)(), langfuseRoutesMiddleware_1.requireAdmin], (0, api_1.asyncHandler)(async (req, res) => {
52
- await (0, langfusePrompts_1.invalidatePromptCache)(req.params.name);
53
- return res.json({ invalidated: true, name: req.params.name });
54
- }));
55
- };
56
- exports.addPromptRoutes = addPromptRoutes;
@@ -1,2 +0,0 @@
1
- import type express from "express";
2
- export declare const addTraceRoutes: (router: express.Application, basePath: string) => void;
@@ -1,30 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.addTraceRoutes = void 0;
4
- const api_1 = require("@terreno/api");
5
- const langfuseClient_1 = require("./langfuseClient");
6
- const langfuseRoutesMiddleware_1 = require("./langfuseRoutesMiddleware");
7
- const addTraceRoutes = (router, basePath) => {
8
- router.get(`${basePath}/traces`, [(0, api_1.authenticateMiddleware)(), langfuseRoutesMiddleware_1.requireAdmin], (0, api_1.asyncHandler)(async (req, res) => {
9
- const client = (0, langfuseClient_1.getLangfuseClient)();
10
- const page = Number(req.query.page) || 1;
11
- const limit = Number(req.query.limit) || 20;
12
- const userId = req.query.userId;
13
- const fromTimestamp = req.query.from ? req.query.from : undefined;
14
- const toTimestamp = req.query.to ? req.query.to : undefined;
15
- const result = await client.api.trace.list({
16
- fromTimestamp,
17
- limit,
18
- page,
19
- toTimestamp,
20
- userId,
21
- });
22
- return res.json({ data: result.data, meta: result.meta });
23
- }));
24
- router.get(`${basePath}/traces/:traceId`, [(0, api_1.authenticateMiddleware)(), langfuseRoutesMiddleware_1.requireAdmin], (0, api_1.asyncHandler)(async (req, res) => {
25
- const client = (0, langfuseClient_1.getLangfuseClient)();
26
- const result = await client.api.trace.get(req.params.traceId);
27
- return res.json(result);
28
- }));
29
- };
30
- exports.addTraceRoutes = addTraceRoutes;
@@ -1,4 +0,0 @@
1
- import { NodeSDK } from "@opentelemetry/sdk-node";
2
- import type { LangfuseAppOptions } from "./langfuseTypes";
3
- export declare const initTracing: (options: LangfuseAppOptions) => NodeSDK;
4
- export declare const shutdownTracing: () => Promise<void>;