@rudderhq/run-intelligence-core 0.4.6-canary.0 → 0.4.6-canary.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/create-agent-benchmark.test.js +1 -2
- package/dist/create-agent-benchmark.test.js.map +1 -1
- package/dist/index.d.ts +0 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +0 -1
- package/dist/index.js.map +1 -1
- package/dist/types.d.ts +1 -2
- package/dist/types.d.ts.map +1 -1
- package/package.json +10 -10
- package/dist/langfuse-scores.d.ts +0 -13
- package/dist/langfuse-scores.d.ts.map +0 -1
- package/dist/langfuse-scores.js +0 -83
- package/dist/langfuse-scores.js.map +0 -1
- package/dist/langfuse-scores.test.d.ts +0 -2
- package/dist/langfuse-scores.test.d.ts.map +0 -1
- package/dist/langfuse-scores.test.js +0 -121
- package/dist/langfuse-scores.test.js.map +0 -1
|
@@ -50,7 +50,6 @@ function makeRunDetail(status = "succeeded") {
|
|
|
50
50
|
agentConfigFingerprint: null,
|
|
51
51
|
runtimeConfigFingerprint: null,
|
|
52
52
|
},
|
|
53
|
-
langfuse: null,
|
|
54
53
|
events: [],
|
|
55
54
|
logContent: null,
|
|
56
55
|
logChunks: [],
|
|
@@ -158,7 +157,7 @@ describe("create-agent benchmark helpers", () => {
|
|
|
158
157
|
role: "engineer",
|
|
159
158
|
},
|
|
160
159
|
}),
|
|
161
|
-
judgeVersion: "
|
|
160
|
+
judgeVersion: "judge-create-agent@production",
|
|
162
161
|
});
|
|
163
162
|
const description = appendCreateAgentBenchmarkMetadata("Create an engineer", metadata);
|
|
164
163
|
expect(extractCreateAgentBenchmarkMetadata(description)).toEqual(metadata);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"create-agent-benchmark.test.js","sourceRoot":"","sources":["../src/create-agent-benchmark.test.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EACL,kCAAkC,EAClC,iCAAiC,EACjC,gCAAgC,EAChC,4BAA4B,EAC5B,mCAAmC,EACnC,oBAAoB,GACrB,MAAM,6BAA6B,CAAC;AAGrC,SAAS,aAAa,CAAC,SAA6C,WAAW;IAC7E,OAAO;QACL,GAAG,EAAE;YACH,EAAE,EAAE,OAAO;YACX,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,aAAa;YACtB,gBAAgB,EAAE,YAAY;YAC9B,aAAa,EAAE,QAAQ;YACvB,MAAM;YACN,SAAS,EAAE,IAAI,IAAI,CAAC,0BAA0B,CAAC;YAC/C,UAAU,EAAE,IAAI,IAAI,CAAC,0BAA0B,CAAC;YAChD,KAAK,EAAE,IAAI;YACX,eAAe,EAAE,IAAI;YACrB,QAAQ,EAAE,CAAC;YACX,MAAM,EAAE,IAAI;YACZ,SAAS,EAAE,IAAI;YACf,UAAU,EAAE,IAAI;YAChB,eAAe,EAAE,IAAI;YACrB,cAAc,EAAE,IAAI;YACpB,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,IAAI;YACZ,QAAQ,EAAE,IAAI;YACd,SAAS,EAAE,IAAI;YACf,aAAa,EAAE,KAAK;YACpB,aAAa,EAAE,IAAI;YACnB,aAAa,EAAE,IAAI;YACnB,SAAS,EAAE,IAAI;YACf,aAAa,EAAE,IAAI;YACnB,UAAU,EAAE,IAAI;YAChB,gBAAgB,EAAE,IAAI;YACtB,YAAY,EAAE,IAAI;YAClB,qBAAqB,EAAE,CAAC;YACxB,eAAe,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE;YACvC,SAAS,EAAE,IAAI,IAAI,CAAC,0BAA0B,CAAC;YAC/C,SAAS,EAAE,IAAI,IAAI,CAAC,0BAA0B,CAAC;SAChD;QACD,SAAS,EAAE,iBAAiB;QAC5B,OAAO,EAAE,QAAQ;QACjB,KAAK,EAAE;YACL,EAAE,EAAE,SAAS;YACb,UAAU,EAAE,OAAO;YACnB,KAAK,EAAE,oBAAoB;SAC5B;QACD,MAAM,EAAE;YACN,gBAAgB,EAAE,aAAa;YAC/B,qBAAqB,EAAE,IAAI;YAC3B,4BAA4B,EAAE,IAAI;YAClC,sBAAsB,EAAE,IAAI;YAC5B,wBAAwB,EAAE,IAAI;SAC/B;QACD,
|
|
1
|
+
{"version":3,"file":"create-agent-benchmark.test.js","sourceRoot":"","sources":["../src/create-agent-benchmark.test.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EACL,kCAAkC,EAClC,iCAAiC,EACjC,gCAAgC,EAChC,4BAA4B,EAC5B,mCAAmC,EACnC,oBAAoB,GACrB,MAAM,6BAA6B,CAAC;AAGrC,SAAS,aAAa,CAAC,SAA6C,WAAW;IAC7E,OAAO;QACL,GAAG,EAAE;YACH,EAAE,EAAE,OAAO;YACX,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,aAAa;YACtB,gBAAgB,EAAE,YAAY;YAC9B,aAAa,EAAE,QAAQ;YACvB,MAAM;YACN,SAAS,EAAE,IAAI,IAAI,CAAC,0BAA0B,CAAC;YAC/C,UAAU,EAAE,IAAI,IAAI,CAAC,0BAA0B,CAAC;YAChD,KAAK,EAAE,IAAI;YACX,eAAe,EAAE,IAAI;YACrB,QAAQ,EAAE,CAAC;YACX,MAAM,EAAE,IAAI;YACZ,SAAS,EAAE,IAAI;YACf,UAAU,EAAE,IAAI;YAChB,eAAe,EAAE,IAAI;YACrB,cAAc,EAAE,IAAI;YACpB,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,IAAI;YACZ,QAAQ,EAAE,IAAI;YACd,SAAS,EAAE,IAAI;YACf,aAAa,EAAE,KAAK;YACpB,aAAa,EAAE,IAAI;YACnB,aAAa,EAAE,IAAI;YACnB,SAAS,EAAE,IAAI;YACf,aAAa,EAAE,IAAI;YACnB,UAAU,EAAE,IAAI;YAChB,gBAAgB,EAAE,IAAI;YACtB,YAAY,EAAE,IAAI;YAClB,qBAAqB,EAAE,CAAC;YACxB,eAAe,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE;YACvC,SAAS,EAAE,IAAI,IAAI,CAAC,0BAA0B,CAAC;YAC/C,SAAS,EAAE,IAAI,IAAI,CAAC,0BAA0B,CAAC;SAChD;QACD,SAAS,EAAE,iBAAiB;QAC5B,OAAO,EAAE,QAAQ;QACjB,KAAK,EAAE;YACL,EAAE,EAAE,SAAS;YACb,UAAU,EAAE,OAAO;YACnB,KAAK,EAAE,oBAAoB;SAC5B;QACD,MAAM,EAAE;YACN,gBAAgB,EAAE,aAAa;YAC/B,qBAAqB,EAAE,IAAI;YAC3B,4BAA4B,EAAE,IAAI;YAClC,sBAAsB,EAAE,IAAI;YAC5B,wBAAwB,EAAE,IAAI;SAC/B;QACD,MAAM,EAAE,EAAE;QACV,UAAU,EAAE,IAAI;QAChB,SAAS,EAAE,EAAE;QACb,UAAU,EAAE;YACV;gBACE,IAAI,EAAE,WAAW;gBACjB,EAAE,EAAE,0BAA0B;gBAC9B,IAAI,EAAE,+CAA+C;gBACrD,KAAK,EAAE,KAAK;aACb;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,EAAE,EAAE,0BAA0B;gBAC9B,IAAI,EAAE,aAAa;gBACnB,MAAM,EAAE,EAAE;gBACV,OAAO,EAAE,SAAS;gBAClB,WAAW,EAAE,EAAE;gBACf,YAAY,EAAE,EAAE;gBAChB,YAAY,EAAE,CAAC;gBACf,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE,KAAK;aACf;SACF;KACF,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,YAA4B,EAAE;IAC/C,OAAO;QACL,EAAE,EAAE,WAAW;QACf,KAAK,EAAE,OAAO;QACd,IAAI,EAAE,KAAK;QACX,MAAM,EAAE,KAAK;QACb,IAAI,EAAE,KAAK;QACX,KAAK,EAAE,0BAA0B;QACjC,IAAI,EAAE,OAAO;QACb,MAAM,EAAE,MAAM;QACd,SAAS,EAAE,WAAW;QACtB,YAAY,EAAE,IAAI;QAClB,gBAAgB,EAAE,aAAa;QAC/B,kBAAkB,EAAE,EAAE;QACtB,aAAa,EAAE,EAAE;QACjB,kBAAkB,EAAE,CAAC;QACrB,iBAAiB,EAAE,CAAC;QACpB,WAAW,EAAE,IAAI;QACjB,QAAQ,EAAE,IAAI;QACd,WAAW,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE,eAAe,EAAE,IAAI,EAAE;QAC9D,eAAe,EAAE,IAAI;QACrB,QAAQ,EAAE,IAAI;QACd,SAAS,EAAE,IAAI,IAAI,CAAC,0BAA0B,CAAC;QAC/C,SAAS,EAAE,IAAI,IAAI,CAAC,0BAA0B,CAAC;QAC/C,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,YAA+B,EAAE;IACrD,OAAO;QACL,EAAE,EAAE,YAAY;QAChB,KAAK,EAAE,OAAO;QACd,IAAI,EAAE,YAAY;QAClB,kBAAkB,EAAE,aAAa;QACjC,iBAAiB,EAAE,IAAI;QACvB,MAAM,EAAE,SAAS;QACjB,OAAO,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE;QACjC,YAAY,EAAE,IAAI;QAClB,eAAe,EAAE,IAAI;QACrB,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,IAAI,IAAI,CAAC,0BAA0B,CAAC;QAC/C,SAAS,EAAE,IAAI,IAAI,CAAC,0BAA0B,CAAC;QAC/C,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,gCAAgC,EAAE,GAAG,EAAE;IAC9C,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;QACtC,MAAM,CAAC,oBAAoB,CAAC;YAC1B,EAAE,EAAE,cAAc;YAClB,MAAM,EAAE,6CAA6C;YACrD,YAAY,EAAE,mBAAmB;YACjC,kBAAkB,EAAE;gBAClB,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE,0BAA0B;gBACjC,gBAAgB,EAAE,KAAK;gBACvB,gBAAgB,EAAE,aAAa;gBAC/B,aAAa,EAAE,CAAC,4BAA4B,CAAC;gBAC7C,mBAAmB,EAAE,IAAI;aAC1B;YACD,QAAQ,EAAE;gBACR,gBAAgB,EAAE,IAAI;gBACtB,mBAAmB,EAAE,CAAC,KAAK,CAAC;aAC7B;YACD,UAAU,EAAE,CAAC,qBAAqB,CAAC;SACpC,CAAC,CAAC,CAAC,aAAa,CAAC;YAChB,EAAE,EAAE,cAAc;YAClB,YAAY,EAAE,mBAAmB;YACjC,kBAAkB,EAAE;gBAClB,IAAI,EAAE,KAAK;gBACX,gBAAgB,EAAE,KAAK;aACxB;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,MAAM,QAAQ,GAAG,iCAAiC,CAAC;YACjD,QAAQ,EAAE,oBAAoB,CAAC;gBAC7B,EAAE,EAAE,iBAAiB;gBACrB,MAAM,EAAE,oBAAoB;gBAC5B,YAAY,EAAE,eAAe;gBAC7B,kBAAkB,EAAE;oBAClB,IAAI,EAAE,UAAU;iBACjB;aACF,CAAC;YACF,YAAY,EAAE,+BAA+B;SAC9C,CAAC,CAAC;QACH,MAAM,WAAW,GAAG,kCAAkC,CAAC,oBAAoB,EAAE,QAAQ,CAAC,CAAC;QACvF,MAAM,CAAC,mCAAmC,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mEAAmE,EAAE,GAAG,EAAE;QAC3E,MAAM,QAAQ,GAAG,oBAAoB,CAAC;YACpC,EAAE,EAAE,cAAc;YAClB,MAAM,EAAE,6CAA6C;YACrD,YAAY,EAAE,mBAAmB;YACjC,kBAAkB,EAAE;gBAClB,IAAI,EAAE,KAAK;gBACX,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE,0BAA0B;gBACjC,gBAAgB,EAAE,KAAK;gBACvB,gBAAgB,EAAE,aAAa;gBAC/B,aAAa,EAAE,CAAC,4BAA4B,CAAC;gBAC7C,mBAAmB,EAAE,IAAI;aAC1B;SACF,CAAC,CAAC;QACH,MAAM,iBAAiB,GAAG,iCAAiC,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9F,MAAM,MAAM,GAAG,4BAA4B,CAAC;YAC1C,QAAQ;YACR,iBAAiB;YACjB,OAAO,EAAE,SAAS;YAClB,SAAS,EAAE,aAAa,EAAE;YAC1B,aAAa,EAAE;gBACb;oBACE,KAAK,EAAE,SAAS,CAAC,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC;oBAChD,MAAM,EAAE;wBACN,gBAAgB,EAAE,aAAa;wBAC/B,SAAS,EAAE,IAAI;wBACf,IAAI,EAAE,YAAY;wBAClB,aAAa,EAAE,CAAC,4BAA4B,CAAC;wBAC7C,OAAO,EAAE,EAAE;wBACX,QAAQ,EAAE,EAAE;qBACb;iBACF;aACF;YACD,gBAAgB,EAAE;gBAChB;oBACE,QAAQ,EAAE,YAAY,EAAE;oBACxB,QAAQ,EAAE,CAAC,SAAS,CAAC;iBACtB;aACF;YACD,WAAW,EAAE,EAAE,GAAG,EAAE,WAAW,EAAE;YACjC,KAAK,EAAE;gBACL,MAAM,EAAE,WAAW;gBACnB,OAAO,EAAE,UAAU;gBACnB,OAAO,EAAE,iCAAiC;gBAC1C,aAAa,EAAE,CAAC;gBAChB,gBAAgB,EAAE,CAAC;gBACnB,yBAAyB,EAAE,CAAC;aAC7B;SACF,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,yBAAyB,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,6BAA6B,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,yBAAyB,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,gCAAgC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1E,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChD,MAAM,CAAC,gCAAgC,CAAC,MAAM,CAAC,MAAM,CAAC,gCAAgC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,GAAG,EAAE;QAClE,MAAM,QAAQ,GAAG,oBAAoB,CAAC;YACpC,EAAE,EAAE,iBAAiB;YACrB,MAAM,EAAE,oBAAoB;YAC5B,YAAY,EAAE,eAAe;YAC7B,kBAAkB,EAAE;gBAClB,IAAI,EAAE,UAAU;aACjB;SACF,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,aAAa,EAAE,CAAC;QAC/B,MAAM,CAAC,UAAU,GAAG,+CAA+C,CAAC;QAEpE,MAAM,MAAM,GAAG,4BAA4B,CAAC;YAC1C,QAAQ;YACR,iBAAiB,EAAE,iCAAiC,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;YACtF,OAAO,EAAE,SAAS;YAClB,SAAS,EAAE,MAAM;YACjB,aAAa,EAAE,EAAE;YACjB,gBAAgB,EAAE,EAAE;YACpB,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,mCAAmC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC7E,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,gCAAgC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1E,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yEAAyE,EAAE,GAAG,EAAE;QACjF,MAAM,QAAQ,GAAG,oBAAoB,CAAC;YACpC,EAAE,EAAE,iBAAiB;YACrB,MAAM,EAAE,oBAAoB;YAC5B,YAAY,EAAE,eAAe;YAC7B,kBAAkB,EAAE;gBAClB,IAAI,EAAE,UAAU;gBAChB,KAAK,EAAE,iBAAiB;gBACxB,gBAAgB,EAAE,aAAa;aAChC;SACF,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;QAExC,MAAM,MAAM,GAAG,4BAA4B,CAAC;YAC1C,QAAQ;YACR,iBAAiB,EAAE,iCAAiC,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;YACtF,OAAO,EAAE,SAAS;YAClB,SAAS,EAAE,MAAM;YACjB,aAAa,EAAE;gBACb;oBACE,KAAK,EAAE,SAAS,CAAC;wBACf,IAAI,EAAE,UAAU;wBAChB,KAAK,EAAE,iBAAiB;wBACxB,gBAAgB,EAAE,aAAa;qBAChC,CAAC;oBACF,MAAM,EAAE;wBACN,gBAAgB,EAAE,aAAa;wBAC/B,SAAS,EAAE,IAAI;wBACf,IAAI,EAAE,YAAY;wBAClB,aAAa,EAAE,EAAE;wBACjB,OAAO,EAAE,EAAE;wBACX,QAAQ,EAAE,EAAE;qBACb;iBACF;aACF;YACD,gBAAgB,EAAE,EAAE;YACpB,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,8BAA8B,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxE,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACxD,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAC;IACnE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/dist/index.d.ts
CHANGED
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,6BAA6B,CAAC;AAC5C,cAAc,gBAAgB,CAAC;AAC/B,cAAc,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,6BAA6B,CAAC;AAC5C,cAAc,gBAAgB,CAAC;AAC/B,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC;AACpC,cAAc,cAAc,CAAC;AAC7B,cAAc,YAAY,CAAC;AAC3B,cAAc,iBAAiB,CAAC;AAChC,cAAc,YAAY,CAAC"}
|
package/dist/index.js
CHANGED
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,6BAA6B,CAAC;AAC5C,cAAc,gBAAgB,CAAC;AAC/B,cAAc,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,6BAA6B,CAAC;AAC5C,cAAc,gBAAgB,CAAC;AAC/B,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC;AACpC,cAAc,cAAc,CAAC;AAC7B,cAAc,YAAY,CAAC;AAC3B,cAAc,iBAAiB,CAAC;AAChC,cAAc,YAAY,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { TranscriptEntry } from "@rudderhq/agent-runtime-utils";
|
|
2
|
-
import type {
|
|
2
|
+
import type { HeartbeatRun, HeartbeatRunEvent } from "@rudderhq/shared";
|
|
3
3
|
export type RunDiagnosisMode = "auto" | "quick" | "error" | "perf" | "full";
|
|
4
4
|
export type RunFindingSeverity = "info" | "warn" | "error";
|
|
5
5
|
export interface EvalBundle {
|
|
@@ -29,7 +29,6 @@ export interface ObservedRun {
|
|
|
29
29
|
orgName: string | null;
|
|
30
30
|
issue: RunIssueRef | null;
|
|
31
31
|
bundle: EvalBundle;
|
|
32
|
-
langfuse?: ExecutionLangfuseLink | null;
|
|
33
32
|
errorSummary?: string | null;
|
|
34
33
|
skillEvidence?: RunSkillEvidenceMatch | null;
|
|
35
34
|
}
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AACrE,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AACrE,OAAO,KAAK,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAExE,MAAM,MAAM,gBAAgB,GAAG,MAAM,GAAG,OAAO,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;AAC5E,MAAM,MAAM,kBAAkB,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;AAE3D,MAAM,WAAW,UAAU;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,4BAA4B,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,wBAAwB,EAAE,MAAM,GAAG,IAAI,CAAC;CACzC;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,MAAM,MAAM,oBAAoB,GAAG,MAAM,GAAG,QAAQ,CAAC;AAErD,MAAM,WAAW,qBAAqB;IACpC,YAAY,EAAE,oBAAoB,CAAC;IACnC,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,YAAY,CAAC;IAClB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,KAAK,EAAE,WAAW,GAAG,IAAI,CAAC;IAC1B,MAAM,EAAE,UAAU,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,aAAa,CAAC,EAAE,qBAAqB,GAAG,IAAI,CAAC;CAC9C;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACvC,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,iBAAkB,SAAQ,WAAW;IACpD,MAAM,EAAE,iBAAiB,EAAE,CAAC;IAC5B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,EAAE,WAAW,EAAE,CAAC;IACzB,UAAU,EAAE,eAAe,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,IAAI,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,OAAO,CAAC;IACtB,cAAc,EAAE,OAAO,CAAC;IACxB,mBAAmB,EAAE,OAAO,CAAC;IAC7B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,OAAO,CAAC;IAClB,KAAK,EAAE,eAAe,EAAE,CAAC;CAC1B;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,eAAe,EAAE,CAAC;IACzB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE,eAAe,EAAE,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,QAAQ,EAAE,QAAQ,GAAG,OAAO,GAAG,aAAa,GAAG,UAAU,GAAG,QAAQ,CAAC;IACrE,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,6BAA6B,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7C,QAAQ,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAClD;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,gBAAgB,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,UAAU,EAAE,CAAC;IACvB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,OAAO,EAAE,UAAU,CAAC;CACrB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,YAAY,CAAC;IACnB,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,KAAK,CAAC;QACZ,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;QAC7B,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;QAC9B,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,YAAa,SAAQ,WAAW;CAAG"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rudderhq/run-intelligence-core",
|
|
3
|
-
"version": "0.4.6-canary.
|
|
3
|
+
"version": "0.4.6-canary.2",
|
|
4
4
|
"license": "SEE LICENSE IN LICENSE",
|
|
5
5
|
"homepage": "https://github.com/Undertone0809/rudder",
|
|
6
6
|
"bugs": {
|
|
@@ -49,15 +49,15 @@
|
|
|
49
49
|
"test": "vitest run --config vitest.config.ts"
|
|
50
50
|
},
|
|
51
51
|
"dependencies": {
|
|
52
|
-
"@rudderhq/agent-runtime-claude-local": "0.4.6-canary.
|
|
53
|
-
"@rudderhq/agent-runtime-codex-local": "0.4.6-canary.
|
|
54
|
-
"@rudderhq/agent-runtime-cursor-local": "0.4.6-canary.
|
|
55
|
-
"@rudderhq/agent-runtime-gemini-local": "0.4.6-canary.
|
|
56
|
-
"@rudderhq/agent-runtime-openclaw-gateway": "0.4.6-canary.
|
|
57
|
-
"@rudderhq/agent-runtime-opencode-local": "0.4.6-canary.
|
|
58
|
-
"@rudderhq/agent-runtime-pi-local": "0.4.6-canary.
|
|
59
|
-
"@rudderhq/agent-runtime-utils": "0.4.6-canary.
|
|
60
|
-
"@rudderhq/shared": "0.4.6-canary.
|
|
52
|
+
"@rudderhq/agent-runtime-claude-local": "0.4.6-canary.2",
|
|
53
|
+
"@rudderhq/agent-runtime-codex-local": "0.4.6-canary.2",
|
|
54
|
+
"@rudderhq/agent-runtime-cursor-local": "0.4.6-canary.2",
|
|
55
|
+
"@rudderhq/agent-runtime-gemini-local": "0.4.6-canary.2",
|
|
56
|
+
"@rudderhq/agent-runtime-openclaw-gateway": "0.4.6-canary.2",
|
|
57
|
+
"@rudderhq/agent-runtime-opencode-local": "0.4.6-canary.2",
|
|
58
|
+
"@rudderhq/agent-runtime-pi-local": "0.4.6-canary.2",
|
|
59
|
+
"@rudderhq/agent-runtime-utils": "0.4.6-canary.2",
|
|
60
|
+
"@rudderhq/shared": "0.4.6-canary.2"
|
|
61
61
|
},
|
|
62
62
|
"devDependencies": {
|
|
63
63
|
"@types/node": "^24.12.0",
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
import type { ExecutionLangfuseLink } from "@rudderhq/shared";
|
|
2
|
-
import type { ObservedRunDetail, RunDiagnosis } from "./types.js";
|
|
3
|
-
export interface LangfuseScoreDraft {
|
|
4
|
-
name: "run_health" | "failure_taxonomy" | "task_outcome" | "budget_guardrail" | "cost_efficiency" | "recovery_success";
|
|
5
|
-
value: boolean | number | string;
|
|
6
|
-
comment?: string;
|
|
7
|
-
metadata?: Record<string, unknown>;
|
|
8
|
-
}
|
|
9
|
-
export declare function buildLangfuseRunScores(detail: ObservedRunDetail, diagnosis: RunDiagnosis): LangfuseScoreDraft[];
|
|
10
|
-
export type LangfuseAnnotatedRun = {
|
|
11
|
-
langfuse?: ExecutionLangfuseLink | null;
|
|
12
|
-
};
|
|
13
|
-
//# sourceMappingURL=langfuse-scores.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"langfuse-scores.d.ts","sourceRoot":"","sources":["../src/langfuse-scores.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAElE,MAAM,WAAW,kBAAkB;IACjC,IAAI,EACA,YAAY,GACZ,kBAAkB,GAClB,cAAc,GACd,kBAAkB,GAClB,iBAAiB,GACjB,kBAAkB,CAAC;IACvB,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AA2CD,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,iBAAiB,EAAE,SAAS,EAAE,YAAY,GAAG,kBAAkB,EAAE,CA2C/G;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,QAAQ,CAAC,EAAE,qBAAqB,GAAG,IAAI,CAAC;CACzC,CAAC"}
|
package/dist/langfuse-scores.js
DELETED
|
@@ -1,83 +0,0 @@
|
|
|
1
|
-
function collectErrorText(detail) {
|
|
2
|
-
return [
|
|
3
|
-
detail.run.error ?? "",
|
|
4
|
-
detail.run.errorCode ?? "",
|
|
5
|
-
detail.run.stderrExcerpt ?? "",
|
|
6
|
-
...detail.events.map((event) => event.message ?? ""),
|
|
7
|
-
].join("\n");
|
|
8
|
-
}
|
|
9
|
-
function isBudgetGuardrail(detail) {
|
|
10
|
-
return /budget|hard[-\s]?limit|token burn|auto-pause/i.test(collectErrorText(detail));
|
|
11
|
-
}
|
|
12
|
-
function resolveTaskOutcome(detail) {
|
|
13
|
-
switch (detail.run.status) {
|
|
14
|
-
case "succeeded":
|
|
15
|
-
return 1;
|
|
16
|
-
case "cancelled":
|
|
17
|
-
return 0;
|
|
18
|
-
case "failed":
|
|
19
|
-
case "timed_out":
|
|
20
|
-
return 0;
|
|
21
|
-
default:
|
|
22
|
-
return 0;
|
|
23
|
-
}
|
|
24
|
-
}
|
|
25
|
-
function resolveCostEfficiency(detail, diagnosis) {
|
|
26
|
-
const costUsd = Number(diagnosis.metrics.costUsd ?? detail.run.usageJson?.costUsd ?? 0);
|
|
27
|
-
if (detail.run.status !== "succeeded")
|
|
28
|
-
return "n/a";
|
|
29
|
-
if (costUsd <= 1)
|
|
30
|
-
return "low";
|
|
31
|
-
if (costUsd <= 5)
|
|
32
|
-
return "medium";
|
|
33
|
-
return "high";
|
|
34
|
-
}
|
|
35
|
-
function resolveRecoverySuccess(detail) {
|
|
36
|
-
const recovery = detail.run.contextSnapshot?.recovery;
|
|
37
|
-
if (!recovery || typeof recovery !== "object")
|
|
38
|
-
return null;
|
|
39
|
-
return detail.run.status === "succeeded";
|
|
40
|
-
}
|
|
41
|
-
export function buildLangfuseRunScores(detail, diagnosis) {
|
|
42
|
-
const scores = [
|
|
43
|
-
{
|
|
44
|
-
name: "run_health",
|
|
45
|
-
value: detail.run.status === "succeeded",
|
|
46
|
-
comment: diagnosis.summary,
|
|
47
|
-
metadata: {
|
|
48
|
-
status: detail.run.status,
|
|
49
|
-
failureTaxonomy: diagnosis.failureTaxonomy,
|
|
50
|
-
},
|
|
51
|
-
},
|
|
52
|
-
{
|
|
53
|
-
name: "failure_taxonomy",
|
|
54
|
-
value: diagnosis.failureTaxonomy,
|
|
55
|
-
comment: detail.run.error ?? diagnosis.summary,
|
|
56
|
-
},
|
|
57
|
-
{
|
|
58
|
-
name: "task_outcome",
|
|
59
|
-
value: resolveTaskOutcome(detail),
|
|
60
|
-
comment: detail.issue?.identifier ?? detail.issue?.title ?? "No linked issue",
|
|
61
|
-
},
|
|
62
|
-
{
|
|
63
|
-
name: "budget_guardrail",
|
|
64
|
-
value: isBudgetGuardrail(detail),
|
|
65
|
-
comment: isBudgetGuardrail(detail) ? "Run was blocked or degraded by budget enforcement." : "No budget guardrail detected.",
|
|
66
|
-
},
|
|
67
|
-
{
|
|
68
|
-
name: "cost_efficiency",
|
|
69
|
-
value: resolveCostEfficiency(detail, diagnosis),
|
|
70
|
-
comment: `costUsd=${Number(diagnosis.metrics.costUsd ?? detail.run.usageJson?.costUsd ?? 0).toFixed(2)}`,
|
|
71
|
-
},
|
|
72
|
-
];
|
|
73
|
-
const recoverySuccess = resolveRecoverySuccess(detail);
|
|
74
|
-
if (recoverySuccess !== null) {
|
|
75
|
-
scores.push({
|
|
76
|
-
name: "recovery_success",
|
|
77
|
-
value: recoverySuccess,
|
|
78
|
-
comment: recoverySuccess ? "Recovery run completed successfully." : "Recovery run did not complete successfully.",
|
|
79
|
-
});
|
|
80
|
-
}
|
|
81
|
-
return scores;
|
|
82
|
-
}
|
|
83
|
-
//# sourceMappingURL=langfuse-scores.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"langfuse-scores.js","sourceRoot":"","sources":["../src/langfuse-scores.ts"],"names":[],"mappings":"AAgBA,SAAS,gBAAgB,CAAC,MAAyB;IACjD,OAAO;QACL,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;QACtB,MAAM,CAAC,GAAG,CAAC,SAAS,IAAI,EAAE;QAC1B,MAAM,CAAC,GAAG,CAAC,aAAa,IAAI,EAAE;QAC9B,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC;KACrD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,iBAAiB,CAAC,MAAyB;IAClD,OAAO,+CAA+C,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC;AACxF,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAyB;IACnD,QAAQ,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;QAC1B,KAAK,WAAW;YACd,OAAO,CAAC,CAAC;QACX,KAAK,WAAW;YACd,OAAO,CAAC,CAAC;QACX,KAAK,QAAQ,CAAC;QACd,KAAK,WAAW;YACd,OAAO,CAAC,CAAC;QACX;YACE,OAAO,CAAC,CAAC;IACb,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAAC,MAAyB,EAAE,SAAuB;IAC/E,MAAM,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,IAAI,CAAC,CAAC,CAAC;IACxF,IAAI,MAAM,CAAC,GAAG,CAAC,MAAM,KAAK,WAAW;QAAE,OAAO,KAAK,CAAC;IACpD,IAAI,OAAO,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/B,IAAI,OAAO,IAAI,CAAC;QAAE,OAAO,QAAQ,CAAC;IAClC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,sBAAsB,CAAC,MAAyB;IACvD,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,eAAe,EAAE,QAAQ,CAAC;IACtD,IAAI,CAAC,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3D,OAAO,MAAM,CAAC,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,MAAyB,EAAE,SAAuB;IACvF,MAAM,MAAM,GAAyB;QACnC;YACE,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,KAAK,WAAW;YACxC,OAAO,EAAE,SAAS,CAAC,OAAO;YAC1B,QAAQ,EAAE;gBACR,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM;gBACzB,eAAe,EAAE,SAAS,CAAC,eAAe;aAC3C;SACF;QACD;YACE,IAAI,EAAE,kBAAkB;YACxB,KAAK,EAAE,SAAS,CAAC,eAAe;YAChC,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,SAAS,CAAC,OAAO;SAC/C;QACD;YACE,IAAI,EAAE,cAAc;YACpB,KAAK,EAAE,kBAAkB,CAAC,MAAM,CAAC;YACjC,OAAO,EAAE,MAAM,CAAC,KAAK,EAAE,UAAU,IAAI,MAAM,CAAC,KAAK,EAAE,KAAK,IAAI,iBAAiB;SAC9E;QACD;YACE,IAAI,EAAE,kBAAkB;YACxB,KAAK,EAAE,iBAAiB,CAAC,MAAM,CAAC;YAChC,OAAO,EAAE,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,oDAAoD,CAAC,CAAC,CAAC,+BAA+B;SAC5H;QACD;YACE,IAAI,EAAE,iBAAiB;YACvB,KAAK,EAAE,qBAAqB,CAAC,MAAM,EAAE,SAAS,CAAC;YAC/C,OAAO,EAAE,WAAW,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;SACzG;KACF,CAAC;IAEF,MAAM,eAAe,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAC;IACvD,IAAI,eAAe,KAAK,IAAI,EAAE,CAAC;QAC7B,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,kBAAkB;YACxB,KAAK,EAAE,eAAe;YACtB,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,sCAAsC,CAAC,CAAC,CAAC,6CAA6C;SAClH,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"langfuse-scores.test.d.ts","sourceRoot":"","sources":["../src/langfuse-scores.test.ts"],"names":[],"mappings":""}
|
|
@@ -1,121 +0,0 @@
|
|
|
1
|
-
import { describe, expect, it } from "vitest";
|
|
2
|
-
import { diagnoseRun } from "./diagnosis.js";
|
|
3
|
-
import { buildLangfuseRunScores } from "./langfuse-scores.js";
|
|
4
|
-
import { observedRunFromFilesystem } from "./loaders/rudder.js";
|
|
5
|
-
function makeRun(overrides = {}) {
|
|
6
|
-
return {
|
|
7
|
-
id: "run-1",
|
|
8
|
-
orgId: "org-1",
|
|
9
|
-
agentId: "agent-1",
|
|
10
|
-
invocationSource: "on_demand",
|
|
11
|
-
triggerDetail: "manual",
|
|
12
|
-
status: "failed",
|
|
13
|
-
startedAt: new Date("2026-04-08T10:00:00.000Z"),
|
|
14
|
-
finishedAt: new Date("2026-04-08T10:02:00.000Z"),
|
|
15
|
-
error: "budget hard limit exceeded",
|
|
16
|
-
wakeupRequestId: null,
|
|
17
|
-
exitCode: 1,
|
|
18
|
-
signal: null,
|
|
19
|
-
usageJson: { inputTokens: 42, outputTokens: 12, costUsd: 0.12 },
|
|
20
|
-
resultJson: null,
|
|
21
|
-
sessionIdBefore: null,
|
|
22
|
-
sessionIdAfter: null,
|
|
23
|
-
logStore: "local_file",
|
|
24
|
-
logRef: "run-1.ndjson",
|
|
25
|
-
logBytes: 100,
|
|
26
|
-
logSha256: null,
|
|
27
|
-
logCompressed: false,
|
|
28
|
-
stdoutExcerpt: null,
|
|
29
|
-
stderrExcerpt: "budget hard limit exceeded",
|
|
30
|
-
errorCode: "budget_limit",
|
|
31
|
-
externalRunId: null,
|
|
32
|
-
processPid: null,
|
|
33
|
-
processStartedAt: null,
|
|
34
|
-
retryOfRunId: null,
|
|
35
|
-
processLossRetryCount: 0,
|
|
36
|
-
contextSnapshot: { issueId: "issue-1" },
|
|
37
|
-
createdAt: new Date("2026-04-08T10:00:00.000Z"),
|
|
38
|
-
updatedAt: new Date("2026-04-08T10:02:00.000Z"),
|
|
39
|
-
...overrides,
|
|
40
|
-
};
|
|
41
|
-
}
|
|
42
|
-
describe("buildLangfuseRunScores", () => {
|
|
43
|
-
it("maps phase-1 score taxonomy from run diagnosis", () => {
|
|
44
|
-
const detail = observedRunFromFilesystem({
|
|
45
|
-
run: makeRun(),
|
|
46
|
-
agentName: "Budget Agent",
|
|
47
|
-
issue: { id: "issue-1", identifier: "RUD-1", title: "Protect the budget" },
|
|
48
|
-
bundle: {
|
|
49
|
-
agentRuntimeType: "process",
|
|
50
|
-
agentConfigRevisionId: null,
|
|
51
|
-
agentConfigRevisionCreatedAt: null,
|
|
52
|
-
agentConfigFingerprint: null,
|
|
53
|
-
runtimeConfigFingerprint: null,
|
|
54
|
-
},
|
|
55
|
-
events: [
|
|
56
|
-
{
|
|
57
|
-
id: 1,
|
|
58
|
-
orgId: "org-1",
|
|
59
|
-
runId: "run-1",
|
|
60
|
-
agentId: "agent-1",
|
|
61
|
-
seq: 1,
|
|
62
|
-
eventType: "error",
|
|
63
|
-
stream: "system",
|
|
64
|
-
level: "error",
|
|
65
|
-
color: null,
|
|
66
|
-
message: "budget hard limit exceeded",
|
|
67
|
-
payload: null,
|
|
68
|
-
createdAt: new Date("2026-04-08T10:02:00.000Z"),
|
|
69
|
-
},
|
|
70
|
-
],
|
|
71
|
-
logContent: "",
|
|
72
|
-
});
|
|
73
|
-
const diagnosis = diagnoseRun(detail, "error");
|
|
74
|
-
const scores = buildLangfuseRunScores(detail, diagnosis);
|
|
75
|
-
const scoreNames = scores.map((score) => score.name);
|
|
76
|
-
expect(scoreNames).toEqual([
|
|
77
|
-
"run_health",
|
|
78
|
-
"failure_taxonomy",
|
|
79
|
-
"task_outcome",
|
|
80
|
-
"budget_guardrail",
|
|
81
|
-
"cost_efficiency",
|
|
82
|
-
]);
|
|
83
|
-
expect(scores.find((score) => score.name === "run_health")?.value).toBe(false);
|
|
84
|
-
expect(scores.find((score) => score.name === "budget_guardrail")?.value).toBe(true);
|
|
85
|
-
expect(scoreNames.includes("human_intervention_required")).toBe(false);
|
|
86
|
-
});
|
|
87
|
-
it("includes recovery_success when a recovery run is present", () => {
|
|
88
|
-
const detail = observedRunFromFilesystem({
|
|
89
|
-
run: makeRun({
|
|
90
|
-
status: "succeeded",
|
|
91
|
-
error: null,
|
|
92
|
-
errorCode: null,
|
|
93
|
-
contextSnapshot: {
|
|
94
|
-
issueId: "issue-1",
|
|
95
|
-
recovery: {
|
|
96
|
-
originalRunId: "run-0",
|
|
97
|
-
failureKind: "adapter_failed",
|
|
98
|
-
failureSummary: "Previous run failed",
|
|
99
|
-
recoveryTrigger: "automatic",
|
|
100
|
-
recoveryMode: "continue_preferred",
|
|
101
|
-
},
|
|
102
|
-
},
|
|
103
|
-
}),
|
|
104
|
-
agentName: "Recovery Agent",
|
|
105
|
-
bundle: {
|
|
106
|
-
agentRuntimeType: "process",
|
|
107
|
-
agentConfigRevisionId: null,
|
|
108
|
-
agentConfigRevisionCreatedAt: null,
|
|
109
|
-
agentConfigFingerprint: null,
|
|
110
|
-
runtimeConfigFingerprint: null,
|
|
111
|
-
},
|
|
112
|
-
logContent: "",
|
|
113
|
-
});
|
|
114
|
-
const diagnosis = diagnoseRun(detail, "auto");
|
|
115
|
-
const scores = buildLangfuseRunScores(detail, diagnosis);
|
|
116
|
-
expect(scores.find((score) => score.name === "recovery_success")).toMatchObject({
|
|
117
|
-
value: true,
|
|
118
|
-
});
|
|
119
|
-
});
|
|
120
|
-
});
|
|
121
|
-
//# sourceMappingURL=langfuse-scores.test.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"langfuse-scores.test.js","sourceRoot":"","sources":["../src/langfuse-scores.test.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAC;AAEhE,SAAS,OAAO,CAAC,YAAmC,EAAE;IACpD,OAAO;QACL,EAAE,EAAE,OAAO;QACX,KAAK,EAAE,OAAO;QACd,OAAO,EAAE,SAAS;QAClB,gBAAgB,EAAE,WAAW;QAC7B,aAAa,EAAE,QAAQ;QACvB,MAAM,EAAE,QAAQ;QAChB,SAAS,EAAE,IAAI,IAAI,CAAC,0BAA0B,CAAC;QAC/C,UAAU,EAAE,IAAI,IAAI,CAAC,0BAA0B,CAAC;QAChD,KAAK,EAAE,4BAA4B;QACnC,eAAe,EAAE,IAAI;QACrB,QAAQ,EAAE,CAAC;QACX,MAAM,EAAE,IAAI;QACZ,SAAS,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;QAC/D,UAAU,EAAE,IAAI;QAChB,eAAe,EAAE,IAAI;QACrB,cAAc,EAAE,IAAI;QACpB,QAAQ,EAAE,YAAY;QACtB,MAAM,EAAE,cAAc;QACtB,QAAQ,EAAE,GAAG;QACb,SAAS,EAAE,IAAI;QACf,aAAa,EAAE,KAAK;QACpB,aAAa,EAAE,IAAI;QACnB,aAAa,EAAE,4BAA4B;QAC3C,SAAS,EAAE,cAAc;QACzB,aAAa,EAAE,IAAI;QACnB,UAAU,EAAE,IAAI;QAChB,gBAAgB,EAAE,IAAI;QACtB,YAAY,EAAE,IAAI;QAClB,qBAAqB,EAAE,CAAC;QACxB,eAAe,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE;QACvC,SAAS,EAAE,IAAI,IAAI,CAAC,0BAA0B,CAAC;QAC/C,SAAS,EAAE,IAAI,IAAI,CAAC,0BAA0B,CAAC;QAC/C,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,MAAM,GAAG,yBAAyB,CAAC;YACvC,GAAG,EAAE,OAAO,EAAE;YACd,SAAS,EAAE,cAAc;YACzB,KAAK,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,oBAAoB,EAAE;YAC1E,MAAM,EAAE;gBACN,gBAAgB,EAAE,SAAS;gBAC3B,qBAAqB,EAAE,IAAI;gBAC3B,4BAA4B,EAAE,IAAI;gBAClC,sBAAsB,EAAE,IAAI;gBAC5B,wBAAwB,EAAE,IAAI;aAC/B;YACD,MAAM,EAAE;gBACN;oBACE,EAAE,EAAE,CAAC;oBACL,KAAK,EAAE,OAAO;oBACd,KAAK,EAAE,OAAO;oBACd,OAAO,EAAE,SAAS;oBAClB,GAAG,EAAE,CAAC;oBACN,SAAS,EAAE,OAAO;oBAClB,MAAM,EAAE,QAAQ;oBAChB,KAAK,EAAE,OAAO;oBACd,KAAK,EAAE,IAAI;oBACX,OAAO,EAAE,4BAA4B;oBACrC,OAAO,EAAE,IAAI;oBACb,SAAS,EAAE,IAAI,IAAI,CAAC,0BAA0B,CAAC;iBAChD;aACF;YACD,UAAU,EAAE,EAAE;SACf,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC/C,MAAM,MAAM,GAAG,sBAAsB,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAEzD,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACrD,MAAM,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC;YACzB,YAAY;YACZ,kBAAkB;YAClB,cAAc;YACd,kBAAkB;YAClB,iBAAiB;SAClB,CAAC,CAAC;QACH,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,YAAY,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/E,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,kBAAkB,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpF,MAAM,CAAE,UAAuB,CAAC,QAAQ,CAAC,6BAA6B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACvF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,GAAG,EAAE;QAClE,MAAM,MAAM,GAAG,yBAAyB,CAAC;YACvC,GAAG,EAAE,OAAO,CAAC;gBACX,MAAM,EAAE,WAAW;gBACnB,KAAK,EAAE,IAAI;gBACX,SAAS,EAAE,IAAI;gBACf,eAAe,EAAE;oBACf,OAAO,EAAE,SAAS;oBAClB,QAAQ,EAAE;wBACR,aAAa,EAAE,OAAO;wBACtB,WAAW,EAAE,gBAAgB;wBAC7B,cAAc,EAAE,qBAAqB;wBACrC,eAAe,EAAE,WAAW;wBAC5B,YAAY,EAAE,oBAAoB;qBACnC;iBACF;aACF,CAAC;YACF,SAAS,EAAE,gBAAgB;YAC3B,MAAM,EAAE;gBACN,gBAAgB,EAAE,SAAS;gBAC3B,qBAAqB,EAAE,IAAI;gBAC3B,4BAA4B,EAAE,IAAI;gBAClC,sBAAsB,EAAE,IAAI;gBAC5B,wBAAwB,EAAE,IAAI;aAC/B;YACD,UAAU,EAAE,EAAE;SACf,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9C,MAAM,MAAM,GAAG,sBAAsB,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAEzD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,kBAAkB,CAAC,CAAC,CAAC,aAAa,CAAC;YAC9E,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|