@kindgi/guardrails 0.1.2 → 0.1.4-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/engine.js +1 -1
- package/dist/engine.js.map +1 -1
- package/dist/errors.d.ts +13 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/execution-strategy.d.ts +1 -1
- package/dist/execution-strategy.d.ts.map +1 -1
- package/dist/execution-strategy.js +1 -1
- package/dist/execution-strategy.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/judge.d.ts +9 -4
- package/dist/judge.d.ts.map +1 -1
- package/dist/judge.js +82 -16
- package/dist/judge.js.map +1 -1
- package/dist/types.d.ts +14 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +6 -6
- package/src/engine.ts +2 -2
- package/src/errors.ts +14 -0
- package/src/execution-strategy.ts +2 -0
- package/src/index.ts +1 -0
- package/src/judge.ts +108 -19
- package/src/types.ts +14 -0
package/dist/engine.js
CHANGED
|
@@ -12,7 +12,7 @@ function getDefaultStrategyRegistry() {
|
|
|
12
12
|
if (defaultRegistry === undefined) {
|
|
13
13
|
defaultRegistry = createExecutionStrategyRegistry([
|
|
14
14
|
zeroLlmStrategy,
|
|
15
|
-
makeLlmJudgeStrategy((config, capability, trace, bindings) => invokeJudge(config, capability, trace, bindings)),
|
|
15
|
+
makeLlmJudgeStrategy((config, capability, trace, bindings, guardrailId) => invokeJudge(config, capability, trace, bindings, guardrailId)),
|
|
16
16
|
externalStrategy,
|
|
17
17
|
]);
|
|
18
18
|
}
|
package/dist/engine.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"engine.js","sourceRoot":"","sources":["../src/engine.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,iCAAiC;AAKjC,OAAO,EAGL,+BAA+B,EAC/B,gBAAgB,EAChB,oBAAoB,EACpB,eAAe,GAChB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAuB,WAAW,EAAE,MAAM,YAAY,CAAC;AAW9D;;;;GAIG;AACH,IAAI,eAAsD,CAAC;AAC3D,SAAS,0BAA0B;IACjC,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QAClC,eAAe,GAAG,+BAA+B,CAAC;YAChD,eAAe;YACf,oBAAoB,CAAC,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,
|
|
1
|
+
{"version":3,"file":"engine.js","sourceRoot":"","sources":["../src/engine.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,iCAAiC;AAKjC,OAAO,EAGL,+BAA+B,EAC/B,gBAAgB,EAChB,oBAAoB,EACpB,eAAe,GAChB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAuB,WAAW,EAAE,MAAM,YAAY,CAAC;AAW9D;;;;GAIG;AACH,IAAI,eAAsD,CAAC;AAC3D,SAAS,0BAA0B;IACjC,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QAClC,eAAe,GAAG,+BAA+B,CAAC;YAChD,eAAe;YACf,oBAAoB,CAAC,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,EAAE,CACxE,WAAW,CAAC,MAAwB,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,CAAC,CAChF;YACD,gBAAgB;SACjB,CAAC,CAAC;IACL,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,qFAAqF;AACrF,MAAM,UAAU,iBAAiB;IAC/B,OAAO,0BAA0B,EAAE,CAAC,IAAI,EAAE,CAAC;AAC7C,CAAC;AA4BD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,SAAoB,EACpB,MAAqB,EACrB,KAAe,EACf,WAA+B,EAAE;IAEjC,MAAM,KAAK,GAAG,UAAU,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAC3C,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IAE3D,mEAAmE;IACnE,0DAA0D;IAC1D,kEAAkE;IAClE,2CAA2C;IAE3C,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,IAAI,0BAA0B,EAAE,CAAC;IACvE,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAChD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE;gBACL,IAAI,EAAE,mBAAmB;gBACzB,OAAO,EAAE,cAAc,SAAS,CAAC,EAAE,4BAA4B,SAAS,CAAC,IAAI,sCAAsC;gBACnH,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,iBAAiB,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC;aACxE;SACF,CAAC;IACJ,CAAC;IACD,MAAM,eAAe,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;IACpF,IAAI,eAAe,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QACnC,MAAM,GAAG,GAAG,eAAe,CAAC,KAAK,CAAC;QAClC,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE;gBACL,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,GAAG,CAAC,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;gBACvD,WAAW,EAAE,SAAS,CAAC,EAAE;aACY;SACxC,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAgB,eAAe,CAAC,KAAK,CAAC;IAElD,MAAM,MAAM,GAA+B,MAAM,CAAC,MAAM;QACtD,CAAC,CAAC,MAAM;QACR,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;IACrC,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,IAAI,OAAO,CAAC;IAC/C,MAAM,UAAU,GAAqB;QACnC,WAAW,EAAE,SAAS,CAAC,EAAE;QACzB,MAAM;QACN,MAAM;QACN,QAAQ;QACR,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAe;KAC1C,CAAC;IAEF,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,KAAK,SAAS,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACzF,8DAA8D;QAC9D,+DAA+D;QAC/D,SAAS;QACT,MAAM,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC;YAC7B,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,IAAI,EAAE,qBAAqB;YAC3B,OAAO,EAAE,QAAQ;YACjB,OAAO,EAAE;gBACP,OAAO,EAAE,CAAC;gBACV,WAAW,EAAE,SAAS,CAAC,EAAE;gBACzB,aAAa,EAAE,SAAS,CAAC,IAAI;gBAC7B,OAAO,EAAE,SAAS,CAAC,KAAK;gBACxB,SAAS,EAAE,SAAS,CAAC,IAAI;gBACzB,QAAQ;gBACR,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,cAAc,CAAC;gBACxC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,UAAU,EAAE,MAAM,CAAC,UAAU;aAC9B;YACD,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI;gBAC/B,aAAa,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE;aACtC,CAAC;SACH,CAAC,CAAC;IACL,CAAC;IAED,uEAAuE;IACvE,wEAAwE;IACxE,wEAAwE;IACxE,iEAAiE;IACjE,IAAI,QAAQ,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACrD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC7C,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,OAAO;gBACL,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE;oBACL,IAAI,EAAE,gBAAgB;oBACtB,OAAO,EAAE,cAAc,SAAS,CAAC,EAAE,mBAAmB,MAAM,+CAA+C;oBAC3G,WAAW,EAAE,SAAS,CAAC,EAAE;oBACzB,MAAM;iBACP;aACF,CAAC;QACJ,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;QACtE,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YAC3B,iEAAiE;YAChE,UAAsC,CAAC,MAAM,GAAG;gBAC/C,GAAG,UAAU,CAAC,MAAM;gBACpB,UAAU,EAAE;oBACV,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC;oBACvC,kBAAkB,EAAE,OAAO,CAAC,KAAK;iBAClC;aACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;AAC3C,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,UAAgC,EAChC,MAAqB,EACrB,KAAe,EACf,WAA+B,EAAE;IAEjC,OAAO,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;AAC/F,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,QAAsC;IAC/D,MAAM,GAAG,GAAuB,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM;YAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,UAAU,CAAC,SAAoB,EAAE,KAAe;IACvD,MAAM,KAAK,GAAU,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC;IAC3C,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,IAAI,QAAQ,CAAC;IACpC,IAAI,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;QAC9C,OAAO;YACL,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE,cAAc,SAAS,CAAC,EAAE,cAAc;YACjD,WAAW,EAAE,SAAS,CAAC,EAAE;YACzB,MAAM,EAAE,YAAY;SACrB,CAAC;IACJ,CAAC;IACD,IAAI,IAAI,KAAK,cAAc,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QACxD,OAAO;YACL,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE,cAAc,SAAS,CAAC,EAAE,mBAAmB;YACtD,WAAW,EAAE,SAAS,CAAC,EAAE;YACzB,MAAM,EAAE,YAAY;SACrB,CAAC;IACJ,CAAC;IACD,IACE,KAAK,CAAC,MAAM,KAAK,SAAS;QAC1B,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;QACvB,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,EACtE,CAAC;QACD,OAAO;YACL,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE,cAAc,SAAS,CAAC,EAAE,6BAA6B,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE;YACvF,WAAW,EAAE,SAAS,CAAC,EAAE;YACzB,MAAM,EAAE,aAAa;SACtB,CAAC;IACJ,CAAC;IACD,IACE,KAAK,CAAC,KAAK,KAAK,SAAS;QACzB,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;QACtB,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EACnE,CAAC;QACD,OAAO;YACL,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE,cAAc,SAAS,CAAC,EAAE,4BAA4B,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE;YACrF,WAAW,EAAE,SAAS,CAAC,EAAE;YACzB,MAAM,EAAE,YAAY;SACrB,CAAC;IACJ,CAAC;IACD,IACE,KAAK,CAAC,OAAO,KAAK,SAAS;QAC3B,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;QACxB,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,EACvC,CAAC;QACD,OAAO;YACL,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE,cAAc,SAAS,CAAC,EAAE,8BAA8B,KAAK,CAAC,QAAQ,EAAE;YACjF,WAAW,EAAE,SAAS,CAAC,EAAE;YACzB,MAAM,EAAE,cAAc;SACvB,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
|
package/dist/errors.d.ts
CHANGED
|
@@ -3,7 +3,7 @@ import type { GuardrailId } from '@kindgi/types';
|
|
|
3
3
|
* Errors emitted by @kindgi/guardrails. Every variant carries a `code`
|
|
4
4
|
* for pattern matching; messages are human-readable, not API contract.
|
|
5
5
|
*/
|
|
6
|
-
export type GuardrailError = InvalidGuardrailError | InvalidCheckDefinitionError | UnknownCheckError | InvalidCheckConfigError | JudgeMissingError | JudgeRoutingError | ScopeMismatchError | UnknownActionError;
|
|
6
|
+
export type GuardrailError = InvalidGuardrailError | InvalidCheckDefinitionError | UnknownCheckError | InvalidCheckConfigError | JudgeMissingError | JudgeRoutingError | JudgeUsageError | ScopeMismatchError | UnknownActionError;
|
|
7
7
|
/**
|
|
8
8
|
* The judge model couldn't be routed via `@kindgi/capabilities`. Wraps
|
|
9
9
|
* the underlying capability error (usually `capability-unsatisfiable` —
|
|
@@ -15,6 +15,18 @@ export interface JudgeRoutingError {
|
|
|
15
15
|
readonly guardrailId: GuardrailId;
|
|
16
16
|
readonly cause: unknown;
|
|
17
17
|
}
|
|
18
|
+
/**
|
|
19
|
+
* An `llm-judge` guardrail's judge answered, but its call couldn't be
|
|
20
|
+
* recorded in the usage sink (`EvaluationBindings.usage`), retries
|
|
21
|
+
* included. The caller fails what it was doing: an answered call isn't
|
|
22
|
+
* left unrecorded.
|
|
23
|
+
*/
|
|
24
|
+
export interface JudgeUsageError {
|
|
25
|
+
readonly code: 'judge-usage-unrecorded';
|
|
26
|
+
readonly message: string;
|
|
27
|
+
readonly guardrailId: GuardrailId;
|
|
28
|
+
readonly cause: unknown;
|
|
29
|
+
}
|
|
18
30
|
/**
|
|
19
31
|
* The declaration is invalid. From `defineGuardrail` /
|
|
20
32
|
* `validateGuardrailSpec`: it fails the schema, or (`defineGuardrail`)
|
package/dist/errors.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAEjD;;;GAGG;AACH,MAAM,MAAM,cAAc,GACtB,qBAAqB,GACrB,2BAA2B,GAC3B,iBAAiB,GACjB,uBAAuB,GACvB,iBAAiB,GACjB,iBAAiB,GACjB,kBAAkB,GAClB,kBAAkB,CAAC;AAEvB;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAC;IACtC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;CACzB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,IAAI,EAAE,mBAAmB,CAAC;IACnC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,SAAS;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACjF;AAED,iEAAiE;AACjE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,eAAe,CAAC;IAC/B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;;GAMG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,iEAAiE;AACjE,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAC;IACtC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;;;;GAIG;AACH,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,CAAC,IAAI,EAAE,0BAA0B,CAAC;IAC1C,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;CACzB;AAED;;;GAGG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,eAAe,CAAC;IAC/B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;CACnC;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,MAAM,EAAE,YAAY,GAAG,aAAa,GAAG,YAAY,GAAG,cAAc,CAAC;CAC/E"}
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAEjD;;;GAGG;AACH,MAAM,MAAM,cAAc,GACtB,qBAAqB,GACrB,2BAA2B,GAC3B,iBAAiB,GACjB,uBAAuB,GACvB,iBAAiB,GACjB,iBAAiB,GACjB,eAAe,GACf,kBAAkB,GAClB,kBAAkB,CAAC;AAEvB;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAC;IACtC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;CACzB;AAED;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,IAAI,EAAE,wBAAwB,CAAC;IACxC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;CACzB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,IAAI,EAAE,mBAAmB,CAAC;IACnC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,SAAS;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACjF;AAED,iEAAiE;AACjE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,eAAe,CAAC;IAC/B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;;GAMG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,iEAAiE;AACjE,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAC;IACtC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;;;;GAIG;AACH,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,CAAC,IAAI,EAAE,0BAA0B,CAAC;IAC1C,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;CACzB;AAED;;;GAGG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,eAAe,CAAC;IAC/B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;CACnC;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,MAAM,EAAE,YAAY,GAAG,aAAa,GAAG,YAAY,GAAG,cAAc,CAAC;CAC/E"}
|
|
@@ -71,7 +71,7 @@ export declare const externalStrategy: ExecutionStrategy;
|
|
|
71
71
|
* function as a parameter so this module does not import judge.ts; the
|
|
72
72
|
* engine passes `invokeJudge`.
|
|
73
73
|
*/
|
|
74
|
-
export declare function makeLlmJudgeStrategy(invokeJudge: (config: unknown, capability: Capability, trace: RunTrace, bindings: EvaluationBindings) => Promise<CheckResult | {
|
|
74
|
+
export declare function makeLlmJudgeStrategy(invokeJudge: (config: unknown, capability: Capability, trace: RunTrace, bindings: EvaluationBindings, guardrailId?: string) => Promise<CheckResult | {
|
|
75
75
|
readonly error: {
|
|
76
76
|
readonly code: string;
|
|
77
77
|
readonly message: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"execution-strategy.d.ts","sourceRoot":"","sources":["../src/execution-strategy.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAEvD,OAAO,KAAK,EACV,aAAa,EACb,WAAW,EACX,kBAAkB,EAClB,SAAS,EACT,QAAQ,EACT,MAAM,YAAY,CAAC;AAEpB;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,2CAA2C;IAC3C,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,QAAQ,CACN,SAAS,EAAE,SAAS,EACpB,MAAM,EAAE,aAAa,EACrB,KAAK,EAAE,QAAQ,EACf,QAAQ,EAAE,kBAAkB,GAC3B,OAAO,CAAC,cAAc,CAAC,CAAC;CAC5B;AAED;;;;GAIG;AACH,MAAM,MAAM,cAAc,GACtB;IAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAA;CAAE,GACpD;IAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAA;CAAE,CAAC;AAE5D,MAAM,WAAW,aAAa;IAC5B,6EAA6E;IAC7E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACjF,uEAAuE;IACvE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED;;;GAGG;AACH,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,QAAQ,EAAE,iBAAiB,GAAG,IAAI,CAAC;IAC5C,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,GAAG,SAAS,CAAC;IACjD,IAAI,IAAI,SAAS,iBAAiB,EAAE,CAAC;CACtC;AAED,wBAAgB,+BAA+B,CAC7C,IAAI,GAAE,SAAS,iBAAiB,EAAO,GACtC,yBAAyB,CAc3B;AAID,yEAAyE;AACzE,eAAO,MAAM,eAAe,EAAE,iBAkB7B,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,EAAE,iBAmB9B,CAAC;AAEF;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,CACX,MAAM,EAAE,OAAO,EACf,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,QAAQ,EACf,QAAQ,EAAE,kBAAkB,
|
|
1
|
+
{"version":3,"file":"execution-strategy.d.ts","sourceRoot":"","sources":["../src/execution-strategy.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAEvD,OAAO,KAAK,EACV,aAAa,EACb,WAAW,EACX,kBAAkB,EAClB,SAAS,EACT,QAAQ,EACT,MAAM,YAAY,CAAC;AAEpB;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,2CAA2C;IAC3C,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,QAAQ,CACN,SAAS,EAAE,SAAS,EACpB,MAAM,EAAE,aAAa,EACrB,KAAK,EAAE,QAAQ,EACf,QAAQ,EAAE,kBAAkB,GAC3B,OAAO,CAAC,cAAc,CAAC,CAAC;CAC5B;AAED;;;;GAIG;AACH,MAAM,MAAM,cAAc,GACtB;IAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAA;CAAE,GACpD;IAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAA;CAAE,CAAC;AAE5D,MAAM,WAAW,aAAa;IAC5B,6EAA6E;IAC7E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACjF,uEAAuE;IACvE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED;;;GAGG;AACH,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,QAAQ,EAAE,iBAAiB,GAAG,IAAI,CAAC;IAC5C,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,GAAG,SAAS,CAAC;IACjD,IAAI,IAAI,SAAS,iBAAiB,EAAE,CAAC;CACtC;AAED,wBAAgB,+BAA+B,CAC7C,IAAI,GAAE,SAAS,iBAAiB,EAAO,GACtC,yBAAyB,CAc3B;AAID,yEAAyE;AACzE,eAAO,MAAM,eAAe,EAAE,iBAkB7B,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,EAAE,iBAmB9B,CAAC;AAEF;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,CACX,MAAM,EAAE,OAAO,EACf,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,QAAQ,EACf,QAAQ,EAAE,kBAAkB,EAC5B,WAAW,CAAC,EAAE,MAAM,KACjB,OAAO,CACV,WAAW,GAAG;IAAE,QAAQ,CAAC,KAAK,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,CACtF,GACA,iBAAiB,CA+BnB"}
|
|
@@ -82,7 +82,7 @@ export function makeLlmJudgeStrategy(invokeJudge) {
|
|
|
82
82
|
},
|
|
83
83
|
};
|
|
84
84
|
}
|
|
85
|
-
const judged = await invokeJudge(guardrail.config ?? {}, guardrail.judgeCapabilities, trace, bindings);
|
|
85
|
+
const judged = await invokeJudge(guardrail.config ?? {}, guardrail.judgeCapabilities, trace, bindings, guardrail.id);
|
|
86
86
|
if ('error' in judged) {
|
|
87
87
|
return {
|
|
88
88
|
kind: 'err',
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"execution-strategy.js","sourceRoot":"","sources":["../src/execution-strategy.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,iCAAiC;AAsEjC,MAAM,UAAU,+BAA+B,CAC7C,OAAqC,EAAE;IAEvC,MAAM,UAAU,GAAG,IAAI,GAAG,EAA6B,CAAC;IACxD,KAAK,MAAM,CAAC,IAAI,IAAI;QAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAChD,OAAO;QACL,QAAQ,CAAC,QAAQ;YACf,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC1C,CAAC;QACD,GAAG,CAAC,IAAI;YACN,OAAO,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;QACD,IAAI;YACF,OAAO,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QAClC,CAAC;KACF,CAAC;AACJ,CAAC;AAED,gDAAgD;AAEhD,yEAAyE;AACzE,MAAM,CAAC,MAAM,eAAe,GAAsB;IAChD,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,gBAAgB;IACtB,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ;QAC/C,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC1C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO;gBACL,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE;oBACL,IAAI,EAAE,eAAe;oBACrB,OAAO,EAAE,cAAc,SAAS,CAAC,EAAE,oCAAoC,SAAS,CAAC,KAAK,GAAG;iBAC1F;aACF,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,CAAC,SAAS,CAAC,MAAM,IAAI,EAAE,CAAsC,CAAC;QAC7E,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC7D,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IACvC,CAAC;CACF,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAsB;IACjD,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,4CAA4C;IAClD,KAAK,CAAC,QAAQ;QACZ,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE;gBACL,IAAI,EAAE,mBAAmB;gBACzB,OAAO,EACL,gHAAgH;gBAClH,MAAM,EAAE;oBACN;wBACE,IAAI,EAAE,OAAO;wBACb,OAAO,EAAE,+DAA+D;qBACzE;iBACF;aACF;SACF,CAAC;IACJ,CAAC;CACF,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAClC,
|
|
1
|
+
{"version":3,"file":"execution-strategy.js","sourceRoot":"","sources":["../src/execution-strategy.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,iCAAiC;AAsEjC,MAAM,UAAU,+BAA+B,CAC7C,OAAqC,EAAE;IAEvC,MAAM,UAAU,GAAG,IAAI,GAAG,EAA6B,CAAC;IACxD,KAAK,MAAM,CAAC,IAAI,IAAI;QAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAChD,OAAO;QACL,QAAQ,CAAC,QAAQ;YACf,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC1C,CAAC;QACD,GAAG,CAAC,IAAI;YACN,OAAO,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;QACD,IAAI;YACF,OAAO,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QAClC,CAAC;KACF,CAAC;AACJ,CAAC;AAED,gDAAgD;AAEhD,yEAAyE;AACzE,MAAM,CAAC,MAAM,eAAe,GAAsB;IAChD,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,gBAAgB;IACtB,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ;QAC/C,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC1C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO;gBACL,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE;oBACL,IAAI,EAAE,eAAe;oBACrB,OAAO,EAAE,cAAc,SAAS,CAAC,EAAE,oCAAoC,SAAS,CAAC,KAAK,GAAG;iBAC1F;aACF,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,CAAC,SAAS,CAAC,MAAM,IAAI,EAAE,CAAsC,CAAC;QAC7E,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC7D,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IACvC,CAAC;CACF,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAsB;IACjD,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,4CAA4C;IAClD,KAAK,CAAC,QAAQ;QACZ,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE;gBACL,IAAI,EAAE,mBAAmB;gBACzB,OAAO,EACL,gHAAgH;gBAClH,MAAM,EAAE;oBACN;wBACE,IAAI,EAAE,OAAO;wBACb,OAAO,EAAE,+DAA+D;qBACzE;iBACF;aACF;SACF,CAAC;IACJ,CAAC;CACF,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAClC,WAQC;IAED,OAAO;QACL,IAAI,EAAE,WAAW;QACjB,IAAI,EAAE,WAAW;QACjB,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ;YAChD,IAAI,SAAS,CAAC,iBAAiB,KAAK,SAAS,EAAE,CAAC;gBAC9C,OAAO;oBACL,IAAI,EAAE,KAAK;oBACX,KAAK,EAAE;wBACL,IAAI,EAAE,mBAAmB;wBACzB,OAAO,EAAE,wBAAwB,SAAS,CAAC,EAAE,6BAA6B;wBAC1E,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,6BAA6B,EAAE,CAAC;qBACjF;iBACF,CAAC;YACJ,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,WAAW,CAC9B,SAAS,CAAC,MAAM,IAAI,EAAE,EACtB,SAAS,CAAC,iBAAiB,EAC3B,KAAK,EACL,QAAQ,EACR,SAAS,CAAC,EAAE,CACb,CAAC;YACF,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;gBACtB,OAAO;oBACL,IAAI,EAAE,KAAK;oBACX,KAAK,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE;iBAClE,CAAC;YACJ,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;QACvC,CAAC;KACF,CAAC;AACJ,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -12,5 +12,5 @@ export { invokeJudge } from './judge.js';
|
|
|
12
12
|
export type { LlmJudgeConfig } from './judge.js';
|
|
13
13
|
export type { Action, Budget, BuiltInGuardrailKind, BuiltInOnViolation, CheckFunction, CheckRegistry, CheckResult, CodeArtifactRef, EvaluationBindings, EvaluationResult, Guardrail, GuardrailKind, GuardrailSeverity, JsonSchema, ModelCallRecord, NetworkPolicy, OnViolation, RegisteredCheck, RunTrace, RuntimeLimits, SandboxMode, Scope, ScopeWhen, ToolCallRecord, ToolResultRecord, TypedNeeds, } from './types.js';
|
|
14
14
|
export { BUILT_IN_GUARDRAIL_KINDS, BUILT_IN_ON_VIOLATIONS } from './types.js';
|
|
15
|
-
export type { InvalidCheckConfigError, InvalidCheckDefinitionError, InvalidGuardrailError, GuardrailError, JudgeMissingError, JudgeRoutingError, ScopeMismatchError, UnknownActionError, UnknownCheckError, } from './errors.js';
|
|
15
|
+
export type { InvalidCheckConfigError, InvalidCheckDefinitionError, InvalidGuardrailError, GuardrailError, JudgeMissingError, JudgeRoutingError, JudgeUsageError, ScopeMismatchError, UnknownActionError, UnknownCheckError, } from './errors.js';
|
|
16
16
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAC3F,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EAAE,eAAe,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACzF,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,2BAA2B,EAC3B,eAAe,EACf,WAAW,EACX,cAAc,EACd,WAAW,EACX,YAAY,GACb,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EACV,aAAa,EACb,aAAa,EACb,qBAAqB,EACrB,YAAY,GACb,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC5F,YAAY,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EACL,+BAA+B,EAC/B,gBAAgB,EAChB,oBAAoB,EACpB,eAAe,GAChB,MAAM,yBAAyB,CAAC;AACjC,YAAY,EACV,iBAAiB,EACjB,yBAAyB,EACzB,aAAa,EACb,cAAc,GACf,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,YAAY,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,YAAY,EACV,MAAM,EACN,MAAM,EACN,oBAAoB,EACpB,kBAAkB,EAClB,aAAa,EACb,aAAa,EACb,WAAW,EACX,eAAe,EACf,kBAAkB,EAClB,gBAAgB,EAChB,SAAS,EACT,aAAa,EACb,iBAAiB,EACjB,UAAU,EACV,eAAe,EACf,aAAa,EACb,WAAW,EACX,eAAe,EACf,QAAQ,EACR,aAAa,EACb,WAAW,EACX,KAAK,EACL,SAAS,EACT,cAAc,EACd,gBAAgB,EAChB,UAAU,GACX,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAC9E,YAAY,EACV,uBAAuB,EACvB,2BAA2B,EAC3B,qBAAqB,EACrB,cAAc,EACd,iBAAiB,EACjB,iBAAiB,EACjB,kBAAkB,EAClB,kBAAkB,EAClB,iBAAiB,GAClB,MAAM,aAAa,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAC3F,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EAAE,eAAe,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACzF,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,2BAA2B,EAC3B,eAAe,EACf,WAAW,EACX,cAAc,EACd,WAAW,EACX,YAAY,GACb,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EACV,aAAa,EACb,aAAa,EACb,qBAAqB,EACrB,YAAY,GACb,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC5F,YAAY,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EACL,+BAA+B,EAC/B,gBAAgB,EAChB,oBAAoB,EACpB,eAAe,GAChB,MAAM,yBAAyB,CAAC;AACjC,YAAY,EACV,iBAAiB,EACjB,yBAAyB,EACzB,aAAa,EACb,cAAc,GACf,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,YAAY,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,YAAY,EACV,MAAM,EACN,MAAM,EACN,oBAAoB,EACpB,kBAAkB,EAClB,aAAa,EACb,aAAa,EACb,WAAW,EACX,eAAe,EACf,kBAAkB,EAClB,gBAAgB,EAChB,SAAS,EACT,aAAa,EACb,iBAAiB,EACjB,UAAU,EACV,eAAe,EACf,aAAa,EACb,WAAW,EACX,eAAe,EACf,QAAQ,EACR,aAAa,EACb,WAAW,EACX,KAAK,EACL,SAAS,EACT,cAAc,EACd,gBAAgB,EAChB,UAAU,GACX,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAC9E,YAAY,EACV,uBAAuB,EACvB,2BAA2B,EAC3B,qBAAqB,EACrB,cAAc,EACd,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,kBAAkB,EAClB,kBAAkB,EAClB,iBAAiB,GAClB,MAAM,aAAa,CAAC"}
|
package/dist/judge.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Capability } from '@kindgi/capabilities';
|
|
2
|
-
import type { JudgeMissingError, JudgeRoutingError } from './errors.js';
|
|
2
|
+
import type { JudgeMissingError, JudgeRoutingError, JudgeUsageError } from './errors.js';
|
|
3
3
|
import type { CheckResult, EvaluationBindings, RunTrace } from './types.js';
|
|
4
4
|
/**
|
|
5
5
|
* Configuration for an llm-judge guardrail, read from the guardrail's
|
|
@@ -23,9 +23,14 @@ export interface LlmJudgeConfig {
|
|
|
23
23
|
* Invoke an LLM judge to evaluate a guardrail. Resolves a model via
|
|
24
24
|
* the capability router (or uses `bindings.judgeProvider` if set), sends
|
|
25
25
|
* a structured judgment prompt including the run trace + rubric, parses
|
|
26
|
-
* the response.
|
|
26
|
+
* the response. The call is recorded in `bindings.usage` (the cost
|
|
27
|
+
* ledger) before its judgment is used, a call that threw included;
|
|
28
|
+
* `guardrailId` says which guardrail it judged. A sink that fails is
|
|
29
|
+
* tried again; an answered call it still can't record is a
|
|
30
|
+
* `judge-usage-unrecorded` error, not a judgment. A call that threw
|
|
31
|
+
* throws its own error, recorded or not: it was billed nothing.
|
|
27
32
|
*/
|
|
28
|
-
export declare function invokeJudge(config: LlmJudgeConfig, capability: Capability, trace: RunTrace, bindings: EvaluationBindings): Promise<CheckResult | {
|
|
29
|
-
readonly error: JudgeMissingError | JudgeRoutingError;
|
|
33
|
+
export declare function invokeJudge(config: LlmJudgeConfig, capability: Capability, trace: RunTrace, bindings: EvaluationBindings, guardrailId?: string): Promise<CheckResult | {
|
|
34
|
+
readonly error: JudgeMissingError | JudgeRoutingError | JudgeUsageError;
|
|
30
35
|
}>;
|
|
31
36
|
//# sourceMappingURL=judge.d.ts.map
|
package/dist/judge.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../src/judge.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../src/judge.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EACV,UAAU,EAKX,MAAM,sBAAsB,CAAC;AAI9B,OAAO,KAAK,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACzF,OAAO,KAAK,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE5E;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,kEAAkE;IAClE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;;;;OAIG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC;IAChD,wEAAwE;IACxE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,qDAAqD;IACrD,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,WAAW,CAC/B,MAAM,EAAE,cAAc,EACtB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,QAAQ,EACf,QAAQ,EAAE,kBAAkB,EAC5B,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CACR,WAAW,GAAG;IAAE,QAAQ,CAAC,KAAK,EAAE,iBAAiB,GAAG,iBAAiB,GAAG,eAAe,CAAA;CAAE,CAC1F,CA+DA"}
|
package/dist/judge.js
CHANGED
|
@@ -1,37 +1,103 @@
|
|
|
1
1
|
// SPDX-License-Identifier: Apache-2.0
|
|
2
2
|
// Copyright (C) 2026 Kindgi Inc.
|
|
3
|
-
import {
|
|
3
|
+
import { randomUUID } from 'node:crypto';
|
|
4
|
+
import { recordModelUsage, route } from '@kindgi/capabilities';
|
|
5
|
+
import { attemptsOf } from '@kindgi/capabilities/attempts';
|
|
4
6
|
/**
|
|
5
7
|
* Invoke an LLM judge to evaluate a guardrail. Resolves a model via
|
|
6
8
|
* the capability router (or uses `bindings.judgeProvider` if set), sends
|
|
7
9
|
* a structured judgment prompt including the run trace + rubric, parses
|
|
8
|
-
* the response.
|
|
10
|
+
* the response. The call is recorded in `bindings.usage` (the cost
|
|
11
|
+
* ledger) before its judgment is used, a call that threw included;
|
|
12
|
+
* `guardrailId` says which guardrail it judged. A sink that fails is
|
|
13
|
+
* tried again; an answered call it still can't record is a
|
|
14
|
+
* `judge-usage-unrecorded` error, not a judgment. A call that threw
|
|
15
|
+
* throws its own error, recorded or not: it was billed nothing.
|
|
9
16
|
*/
|
|
10
|
-
export async function invokeJudge(config, capability, trace, bindings) {
|
|
17
|
+
export async function invokeJudge(config, capability, trace, bindings, guardrailId) {
|
|
11
18
|
const provider = await resolveJudgeProvider(capability, trace, bindings);
|
|
12
19
|
if ('error' in provider)
|
|
13
20
|
return provider;
|
|
14
21
|
const responseFormat = config.responseFormat ?? 'pass-fail';
|
|
15
22
|
const rubric = config.rubric;
|
|
16
23
|
const prompt = buildJudgePrompt(rubric, responseFormat, trace);
|
|
17
|
-
const
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
24
|
+
const record = (call) => recordJudgeCall(bindings, trace, provider, guardrailId, call);
|
|
25
|
+
const callId = randomUUID();
|
|
26
|
+
const startedAt = Date.now();
|
|
27
|
+
let result;
|
|
28
|
+
try {
|
|
29
|
+
result = await provider.provider.invoke({
|
|
30
|
+
model: provider.model.name,
|
|
31
|
+
messages: [
|
|
32
|
+
{
|
|
33
|
+
role: 'system',
|
|
34
|
+
content: responseFormat === 'pass-fail'
|
|
35
|
+
? 'You are an evaluator. Respond starting with PASS or FAIL on the first line, then a brief reason on the next line. No preamble.'
|
|
36
|
+
: 'You are an evaluator. Respond with a numeric score from 0 to 1 on the first line, then a brief reason on the next line. No preamble.',
|
|
37
|
+
},
|
|
38
|
+
{ role: 'user', content: prompt },
|
|
39
|
+
],
|
|
40
|
+
...(config.temperature !== undefined && { temperature: config.temperature }),
|
|
41
|
+
maxOutputTokens: 256,
|
|
42
|
+
...(bindings.abortSignal !== undefined && { abortSignal: bindings.abortSignal }),
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
catch (cause) {
|
|
46
|
+
const attempts = attemptsOf(cause);
|
|
47
|
+
await record({
|
|
48
|
+
callId,
|
|
49
|
+
status: 'failed',
|
|
50
|
+
error: {
|
|
51
|
+
message: cause instanceof Error ? cause.message : String(cause),
|
|
52
|
+
...(attempts !== undefined && { attempts }),
|
|
25
53
|
},
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
54
|
+
durationMs: Date.now() - startedAt,
|
|
55
|
+
});
|
|
56
|
+
throw cause;
|
|
57
|
+
}
|
|
58
|
+
const { message: _answer, ...used } = result;
|
|
59
|
+
const unrecorded = await record({
|
|
60
|
+
callId,
|
|
61
|
+
status: 'ok',
|
|
62
|
+
result: used,
|
|
63
|
+
durationMs: result.durationMs,
|
|
31
64
|
});
|
|
65
|
+
if (unrecorded !== undefined) {
|
|
66
|
+
return {
|
|
67
|
+
error: {
|
|
68
|
+
code: 'judge-usage-unrecorded',
|
|
69
|
+
message: `The judge's model call couldn't be recorded: ${unrecorded instanceof Error ? unrecorded.message : String(unrecorded)}`,
|
|
70
|
+
guardrailId: (guardrailId ?? '<unknown>'),
|
|
71
|
+
cause: unrecorded,
|
|
72
|
+
},
|
|
73
|
+
};
|
|
74
|
+
}
|
|
32
75
|
const text = result.message.content.trim();
|
|
33
76
|
return parseJudgeResponse(text, responseFormat, config.threshold);
|
|
34
77
|
}
|
|
78
|
+
/**
|
|
79
|
+
* Record a judge call in the usage sink, with the run's identity from the
|
|
80
|
+
* trace, trying a failing sink again. Resolves with the sink's last
|
|
81
|
+
* failure when it couldn't record.
|
|
82
|
+
*/
|
|
83
|
+
async function recordJudgeCall(bindings, trace, provider, guardrailId, call) {
|
|
84
|
+
if (bindings.usage === undefined)
|
|
85
|
+
return;
|
|
86
|
+
const recorded = await recordModelUsage(bindings.usage, {
|
|
87
|
+
...call,
|
|
88
|
+
tenantId: trace.tenantId,
|
|
89
|
+
...(trace.projectId !== undefined && { projectId: trace.projectId }),
|
|
90
|
+
runId: trace.runId,
|
|
91
|
+
...(trace.agentId !== undefined && { agentId: trace.agentId }),
|
|
92
|
+
...(trace.conversationId !== undefined && { conversationId: trace.conversationId }),
|
|
93
|
+
providerId: provider.provider.metadata.id,
|
|
94
|
+
model: provider.model.name,
|
|
95
|
+
...(provider.provider.metadata.fallback === true && { fallback: true }),
|
|
96
|
+
purpose: guardrailId !== undefined ? `guardrail-judge:${guardrailId}` : 'guardrail-judge',
|
|
97
|
+
occurredAt: new Date().toISOString(),
|
|
98
|
+
});
|
|
99
|
+
return recorded.kind === 'err' ? (recorded.error ?? new Error('no detail')) : undefined;
|
|
100
|
+
}
|
|
35
101
|
async function resolveJudgeProvider(capability, trace, bindings) {
|
|
36
102
|
if (bindings.judgeProvider !== undefined && bindings.tenantPolicy !== undefined) {
|
|
37
103
|
// Override under a tenant policy: route over the override alone, so
|
package/dist/judge.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"judge.js","sourceRoot":"","sources":["../src/judge.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,iCAAiC;AAEjC,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"judge.js","sourceRoot":"","sources":["../src/judge.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,iCAAiC;AAEjC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,gBAAgB,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAQ/D,OAAO,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAyB3D;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAsB,EACtB,UAAsB,EACtB,KAAe,EACf,QAA4B,EAC5B,WAAoB;IAIpB,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,UAAU,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;IACzE,IAAI,OAAO,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAEzC,MAAM,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,WAAW,CAAC;IAC5D,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7B,MAAM,MAAM,GAAG,gBAAgB,CAAC,MAAM,EAAE,cAAc,EAAE,KAAK,CAAC,CAAC;IAE/D,MAAM,MAAM,GAAG,CAAC,IAAsB,EAAgC,EAAE,CACtE,eAAe,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC;IAChE,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,IAAI,MAAuB,CAAC;IAC5B,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;YACtC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI;YAC1B,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,QAAQ;oBACd,OAAO,EACL,cAAc,KAAK,WAAW;wBAC5B,CAAC,CAAC,gIAAgI;wBAClI,CAAC,CAAC,sIAAsI;iBAC7I;gBACD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE;aAClC;YACD,GAAG,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS,IAAI,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC;YAC5E,eAAe,EAAE,GAAG;YACpB,GAAG,CAAC,QAAQ,CAAC,WAAW,KAAK,SAAS,IAAI,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC;SACjF,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QACnC,MAAM,MAAM,CAAC;YACX,MAAM;YACN,MAAM,EAAE,QAAQ;YAChB,KAAK,EAAE;gBACL,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;gBAC/D,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,CAAC;aAC5C;YACD,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SACnC,CAAC,CAAC;QACH,MAAM,KAAK,CAAC;IACd,CAAC;IACD,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC;IAC7C,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC;QAC9B,MAAM;QACN,MAAM,EAAE,IAAI;QACZ,MAAM,EAAE,IAAI;QACZ,UAAU,EAAE,MAAM,CAAC,UAAU;KAC9B,CAAC,CAAC;IACH,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO;YACL,KAAK,EAAE;gBACL,IAAI,EAAE,wBAAwB;gBAC9B,OAAO,EAAE,gDAAgD,UAAU,YAAY,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE;gBAChI,WAAW,EAAE,CAAC,WAAW,IAAI,WAAW,CAAgB;gBACxD,KAAK,EAAE,UAAU;aAClB;SACF,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IAC3C,OAAO,kBAAkB,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;AACpE,CAAC;AAQD;;;;GAIG;AACH,KAAK,UAAU,eAAe,CAC5B,QAA4B,EAC5B,KAAe,EACf,QAA0B,EAC1B,WAA+B,EAC/B,IAAsB;IAEtB,IAAI,QAAQ,CAAC,KAAK,KAAK,SAAS;QAAE,OAAO;IACzC,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,QAAQ,CAAC,KAAK,EAAE;QACtD,GAAG,IAAI;QACP,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,GAAG,CAAC,KAAK,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC;QACpE,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,OAA4B,EAAE,CAAC;QACnF,GAAG,CAAC,KAAK,CAAC,cAAc,KAAK,SAAS,IAAI,EAAE,cAAc,EAAE,KAAK,CAAC,cAAc,EAAE,CAAC;QACnF,UAAU,EAAE,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;QACzC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI;QAC1B,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,KAAK,IAAI,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QACvE,OAAO,EAAE,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,mBAAmB,WAAW,EAAE,CAAC,CAAC,CAAC,iBAAiB;QACzF,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACrC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,IAAI,IAAI,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC1F,CAAC;AAQD,KAAK,UAAU,oBAAoB,CACjC,UAAsB,EACtB,KAAe,EACf,QAA4B;IAE5B,IAAI,QAAQ,CAAC,aAAa,KAAK,SAAS,IAAI,QAAQ,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QAChF,oEAAoE;QACpE,qDAAqD;QACrD,MAAM,QAAQ,GAAG,KAAK,CAAC;YACrB,UAAU;YACV,SAAS,EAAE,CAAC,QAAQ,CAAC,aAAa,CAAC;YACnC,YAAY,EAAE,QAAQ,CAAC,YAAY;SACpC,CAAC,CAAC;QACH,IAAI,QAAQ,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAsB;gBAC7B,IAAI,EAAE,sBAAsB;gBAC5B,OAAO,EAAE,2BAA2B,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,qCAAqC,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE;gBACnI,WAAW,EAAE,WAAoB;gBACjC,KAAK,EAAE,QAAQ,CAAC,KAAK;aACtB,CAAC;YACF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;QACxB,CAAC;QACD,OAAO;YACL,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,QAAQ;YACjC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,KAAK;YAC3B,MAAM,EAAE,oCAAoC,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE;SACpE,CAAC;IACJ,CAAC;IACD,IAAI,QAAQ,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QACzC,gEAAgE;QAChE,mEAAmE;QACnE,kEAAkE;QAClE,mCAAmC;QACnC,MAAM,QAAQ,GAAG,QAAQ,CAAC,aAAa,CAAC;QACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,GAAG,GAAsB;gBAC7B,IAAI,EAAE,sBAAsB;gBAC5B,OAAO,EAAE,2BAA2B,QAAQ,CAAC,QAAQ,CAAC,EAAE,uBAAuB;gBAC/E,WAAW,EAAE,WAAoB;gBACjC,KAAK,EAAE;oBACL,IAAI,EAAE,0BAA0B;oBAChC,OAAO,EAAE,6BAA6B;oBACtC,OAAO,EAAE,EAAE;iBACZ;aACF,CAAC;YACF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;QACxB,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,iCAAiC,EAAE,CAAC;IAC/E,CAAC;IACD,IAAI,QAAQ,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;QAC5C,MAAM,GAAG,GAAsB;YAC7B,IAAI,EAAE,eAAe;YACrB,OAAO,EACL,qFAAqF;YACvF,WAAW,EAAE,WAAoB;SAClC,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IACxB,CAAC;IACD,MAAM,QAAQ,GAAG,KAAK,CAAC;QACrB,UAAU;QACV,SAAS,EAAE,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;QACzD,GAAG,CAAC,QAAQ,CAAC,YAAY,KAAK,SAAS,IAAI,EAAE,YAAY,EAAE,QAAQ,CAAC,YAAY,EAAE,CAAC;KACpF,CAAC,CAAC;IACH,IAAI,QAAQ,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAsB;YAC7B,IAAI,EAAE,sBAAsB;YAC5B,OAAO,EAAE,yBAAyB,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE;YAC1D,WAAW,EAAE,WAAoB;YACjC,KAAK,EAAE,QAAQ,CAAC,KAAK;SACtB,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IACxB,CAAC;IACD,OAAO;QACL,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,QAAQ;QACjC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,KAAK;QAC3B,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM;KAC9B,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAc,EAAE,MAA6B,EAAE,KAAe;IACtF,MAAM,KAAK,GAAG;QACZ,sDAAsD;QACtD,EAAE;QACF,gBAAgB;QAChB,MAAM;QACN,EAAE;QACF,oBAAoB;QACpB,KAAK,CAAC,MAAM,IAAI,SAAS;QACzB,EAAE;QACF,oBAAoB;QACpB,KAAK,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;YAC1B,CAAC,CAAC,QAAQ;YACV,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,QAAQ,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;QAC5F,EAAE;QACF,MAAM,KAAK,WAAW;YACpB,CAAC,CAAC,mCAAmC;YACrC,CAAC,CAAC,6CAA6C;KAClD,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,kBAAkB,CACzB,GAAW,EACX,MAA6B,EAC7B,SAA6B;IAE7B,IAAI,MAAM,KAAK,WAAW,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QAChC,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC7B,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YACvD,MAAM,MAAM,GAAgB,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,CAAC;YACjE,OAAO,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QAC5E,CAAC;QACD,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC7B,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YACvD,OAAO;gBACL,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,qBAAqB;gBAClE,aAAa,EAAE,GAAG;aACnB,CAAC;QACJ,CAAC;QACD,OAAO;YACL,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,gDAAgD,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE;YAC1E,aAAa,EAAE,GAAG;SACnB,CAAC;IACJ,CAAC;IACD,eAAe;IACf,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACtD,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IAC3C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO;YACL,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,yCAAyC,SAAS,EAAE;YAC5D,aAAa,EAAE,GAAG;SACnB,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,SAAS,IAAI,GAAG,CAAC;IAChC,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACvD,OAAO;QACL,MAAM,EAAE,KAAK,IAAI,MAAM;QACvB,MAAM,EAAE,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,KAAK,eAAe,MAAM,EAAE;QAClF,aAAa,EAAE,GAAG;QAClB,UAAU,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE;KACzC,CAAC;AACJ,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import type { Capability, ModelProvider, ProviderRegistry, TenantPolicy } from '@kindgi/capabilities';
|
|
1
|
+
import type { Capability, ModelProvider, ProviderRegistry, TenantPolicy, UsageSink } from '@kindgi/capabilities';
|
|
2
2
|
import type { ComplianceProvider } from '@kindgi/compliance';
|
|
3
|
-
import type { AgentId, FlowId, GuardrailId, ProjectId, RunId, TenantId, Timestamp, ToolId } from '@kindgi/types';
|
|
3
|
+
import type { AgentId, FlowId, GuardrailId, OrgId, ProjectId, RunId, TenantId, Timestamp, ToolId } from '@kindgi/types';
|
|
4
4
|
/**
|
|
5
5
|
* How a guardrail is checked. Open string — the engine dispatches
|
|
6
6
|
* on this value through an `ExecutionStrategyRegistry`. Built-in kinds
|
|
@@ -268,6 +268,12 @@ export interface RunTrace {
|
|
|
268
268
|
* Optional; when absent, the violation emit is skipped.
|
|
269
269
|
*/
|
|
270
270
|
readonly projectId?: ProjectId;
|
|
271
|
+
/**
|
|
272
|
+
* The org of that project, when it belongs to one: resolved by the
|
|
273
|
+
* runtime from the project, never from input. Absent when the project
|
|
274
|
+
* has no org.
|
|
275
|
+
*/
|
|
276
|
+
readonly orgId?: OrgId;
|
|
271
277
|
readonly agentId?: AgentId;
|
|
272
278
|
readonly flowId?: FlowId;
|
|
273
279
|
/** Final assistant output text. Present when the run produced text. */
|
|
@@ -337,6 +343,12 @@ export interface EvaluationBindings {
|
|
|
337
343
|
* tests + hermetic pinning; production should route through the registry.
|
|
338
344
|
*/
|
|
339
345
|
readonly judgeProvider?: ModelProvider;
|
|
346
|
+
/**
|
|
347
|
+
* Where llm-judge guardrails record their model calls (the runtime's
|
|
348
|
+
* cost ledger), with the run's identity from the trace, before the
|
|
349
|
+
* judgment is used. Absent: judge calls aren't recorded.
|
|
350
|
+
*/
|
|
351
|
+
readonly usage?: UsageSink;
|
|
340
352
|
/**
|
|
341
353
|
* The tenant's model-routing policy (provider / model allow and deny
|
|
342
354
|
* lists, `regionAllow`, caps). When present, llm-judge models are
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,UAAU,EACV,aAAa,EACb,gBAAgB,EAChB,YAAY,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,UAAU,EACV,aAAa,EACb,gBAAgB,EAChB,YAAY,EACZ,SAAS,EACV,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,KAAK,EACV,OAAO,EACP,MAAM,EACN,WAAW,EACX,KAAK,EACL,SAAS,EACT,KAAK,EACL,QAAQ,EACR,SAAS,EACT,MAAM,EACP,MAAM,eAAe,CAAC;AAEvB;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,aAAa,GAAG,MAAM,CAAC;AAEnC,eAAO,MAAM,wBAAwB,gDAAiD,CAAC;AACvF,MAAM,MAAM,oBAAoB,GAAG,CAAC,OAAO,wBAAwB,CAAC,CAAC,MAAM,CAAC,CAAC;AAE7E,MAAM,MAAM,iBAAiB,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,UAAU,CAAC;AAEvE;;;;;;;;;GASG;AACH,MAAM,MAAM,WAAW,GAAG,MAAM,CAAC;AAEjC,eAAO,MAAM,sBAAsB,kEAMzB,CAAC;AACX,MAAM,MAAM,kBAAkB,GAAG,CAAC,OAAO,sBAAsB,CAAC,CAAC,MAAM,CAAC,CAAC;AAEzE,MAAM,MAAM,SAAS,GAAG,QAAQ,GAAG,SAAS,GAAG,cAAc,CAAC;AAE9D,MAAM,WAAW,MAAM;IACrB,QAAQ,CAAC,cAAc,EAAE,WAAW,CAAC;IACrC,QAAQ,CAAC,KAAK,CAAC,EAAE;QAAE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IAClD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;CAClC;AAED,MAAM,WAAW,KAAK;IACpB,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,CAAC;IAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,OAAO,EAAE,CAAC;IACrC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,QAAQ,EAAE,CAAC;CACxC;AAED,MAAM,WAAW,MAAM;IACrB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;CAChC;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,kBAAkB,GAAG,QAAQ,CAAC;AAEjE,6EAA6E;AAC7E,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,mEAAmE;AACnE,MAAM,MAAM,aAAa,GACrB;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACzB;IAAE,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAA;CAAE,GACjE;IAAE,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAA;CAAE,CAAC;AAEtC;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AAE3D;;;GAGG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,GAAG,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;IACpD,QAAQ,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;IACxD,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;IACvD,QAAQ,CAAC,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1C,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACvC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,MAAM,eAAe,GACvB;IACE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;IACrB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;CAClC,GACD;IACE,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B,CAAC;AAEN;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,WAAW,SAAS;IACxB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,EAAE,WAAW,CAAC;IACzB,qEAAqE;IACrE,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,kIAAkI;IAClI,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;;OAOG;IACH,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B;;;;;;;OAOG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;;;;OAMG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IACpD;;;;;;;OAOG;IACH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;;;;;OAKG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IACtC;;;;;OAKG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC;IACvB;;;;OAIG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;OAKG;IACH,QAAQ,CAAC,iBAAiB,CAAC,EAAE,UAAU,CAAC;IACxC,6CAA6C;IAC7C,QAAQ,CAAC,OAAO,CAAC,EAAE,WAAW,CAAC;IAC/B,mEAAmE;IACnE,QAAQ,CAAC,MAAM,CAAC,EAAE,aAAa,CAAC;IAChC,qEAAqE;IACrE,QAAQ,CAAC,OAAO,CAAC,EAAE,aAAa,CAAC;IACjC;;;OAGG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,UAAU,CAAC;IAChC;;;OAGG;IACH,QAAQ,CAAC,eAAe,CAAC,EAAE,eAAe,CAAC;CAC5C;AAED;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IACtD,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;CACxB;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;CACxB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC5B;;;;;OAKG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,CAAC;IAC/B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC;IACvB,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;IAC3B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,uEAAuE;IACvE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,SAAS,cAAc,EAAE,CAAC;IAC9C,QAAQ,CAAC,WAAW,EAAE,SAAS,gBAAgB,EAAE,CAAC;IAClD,QAAQ,CAAC,UAAU,EAAE,SAAS,eAAe,EAAE,CAAC;IAChD;;;;OAIG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;OAIG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC9C;;;OAGG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC;;;OAGG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,8CAA8C;IAC9C,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,2CAA2C;IAC3C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B;;;OAGG;IACH,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,CAAC;IAChC,mFAAmF;IACnF,QAAQ,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CACzD;AAED;;;;;;GAMG;AACH,MAAM,WAAW,kBAAkB;IACjC,kFAAkF;IAClF,QAAQ,CAAC,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IAC7C;;;;;OAKG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;IACnC;;;;OAIG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,kBAAkB,CAAC;IACzC;;;OAGG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;IACvC;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,CAAC;IAC3B;;;;;OAKG;IACH,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY,CAAC;IACrC;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,yBAAyB,EAAE,yBAAyB,CAAC;IAClF;;;;;;OAMG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,qBAAqB,EAAE,qBAAqB,CAAC;CACxE;AAED,2CAA2C;AAC3C,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,kEAAkE;IAClE,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,sEAAsE;IACtE,QAAQ,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CACzD;AAED,kFAAkF;AAClF,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,iFAAiF;IACjF,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,MAAM,CAAC;IACtC,QAAQ,CAAC,QAAQ,EAAE,iBAAiB,CAAC;IACrC,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC;CACxB;AAED;;;;GAIG;AACH,MAAM,MAAM,aAAa,GAAG,CAC1B,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EACzC,KAAK,EAAE,QAAQ,EACf,QAAQ,EAAE,kBAAkB,KACzB,OAAO,CAAC,WAAW,CAAC,CAAC;AAE1B;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;IACjC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,MAAM,GAAG,SAAS,CAAC;CACnE;AAED;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,eAAe,GAAG,IAAI,CAAC;IACvC,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,eAAe,GAAG,SAAS,CAAC;IAC7C,IAAI,IAAI,SAAS,eAAe,EAAE,CAAC;CACpC"}
|
package/dist/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,iCAAiC;
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,iCAAiC;AAqCjC,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,UAAU,EAAE,WAAW,EAAE,UAAU,CAAU,CAAC;AAiBvF,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACpC,MAAM;IACN,OAAO;IACP,UAAU;IACV,UAAU;IACV,YAAY;CACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kindgi/guardrails",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.4-rc.0",
|
|
4
4
|
"description": "Runtime + CI enforcement of agent-behavior guardrails for Kindgi. Same declaration runs at runtime (halt / retry / escalate / log-only / compensate on violation) and in CI (fails the build) — no drift between test and prod. Zero-LLM checks (must-cite, never-call-tool, max-tool-calls, output-matches, tool-order, required-substring, forbidden-substring) are the default fast path; LLM-judge guardrails route via @kindgi/capabilities with explicit budget declarations. Execution strategies and action handlers are pluggable registries, and failed checks optionally emit compliance evidence.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -30,10 +30,10 @@
|
|
|
30
30
|
"type-manifest.json"
|
|
31
31
|
],
|
|
32
32
|
"dependencies": {
|
|
33
|
-
"@kindgi/capabilities": "0.1.
|
|
34
|
-
"@kindgi/compliance": "0.1.
|
|
35
|
-
"@kindgi/schema": "0.1.
|
|
36
|
-
"@kindgi/types": "0.1.
|
|
33
|
+
"@kindgi/capabilities": "0.1.4-rc.0",
|
|
34
|
+
"@kindgi/compliance": "0.1.4-rc.0",
|
|
35
|
+
"@kindgi/schema": "0.1.4-rc.0",
|
|
36
|
+
"@kindgi/types": "0.1.4-rc.0",
|
|
37
37
|
"ajv": "^8.17.1",
|
|
38
38
|
"ajv-formats": "^3.0.1"
|
|
39
39
|
},
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
}
|
|
47
47
|
},
|
|
48
48
|
"devDependencies": {
|
|
49
|
-
"@kindgi/specs": "0.1.
|
|
49
|
+
"@kindgi/specs": "0.1.4-rc.0",
|
|
50
50
|
"@types/node": "^22.10.5",
|
|
51
51
|
"typescript": "^5.7.3",
|
|
52
52
|
"vitest": "^2.1.8",
|
package/src/engine.ts
CHANGED
|
@@ -33,8 +33,8 @@ function getDefaultStrategyRegistry(): ExecutionStrategyRegistry {
|
|
|
33
33
|
if (defaultRegistry === undefined) {
|
|
34
34
|
defaultRegistry = createExecutionStrategyRegistry([
|
|
35
35
|
zeroLlmStrategy,
|
|
36
|
-
makeLlmJudgeStrategy((config, capability, trace, bindings) =>
|
|
37
|
-
invokeJudge(config as LlmJudgeConfig, capability, trace, bindings),
|
|
36
|
+
makeLlmJudgeStrategy((config, capability, trace, bindings, guardrailId) =>
|
|
37
|
+
invokeJudge(config as LlmJudgeConfig, capability, trace, bindings, guardrailId),
|
|
38
38
|
),
|
|
39
39
|
externalStrategy,
|
|
40
40
|
]);
|
package/src/errors.ts
CHANGED
|
@@ -14,6 +14,7 @@ export type GuardrailError =
|
|
|
14
14
|
| InvalidCheckConfigError
|
|
15
15
|
| JudgeMissingError
|
|
16
16
|
| JudgeRoutingError
|
|
17
|
+
| JudgeUsageError
|
|
17
18
|
| ScopeMismatchError
|
|
18
19
|
| UnknownActionError;
|
|
19
20
|
|
|
@@ -29,6 +30,19 @@ export interface JudgeRoutingError {
|
|
|
29
30
|
readonly cause: unknown;
|
|
30
31
|
}
|
|
31
32
|
|
|
33
|
+
/**
|
|
34
|
+
* An `llm-judge` guardrail's judge answered, but its call couldn't be
|
|
35
|
+
* recorded in the usage sink (`EvaluationBindings.usage`), retries
|
|
36
|
+
* included. The caller fails what it was doing: an answered call isn't
|
|
37
|
+
* left unrecorded.
|
|
38
|
+
*/
|
|
39
|
+
export interface JudgeUsageError {
|
|
40
|
+
readonly code: 'judge-usage-unrecorded';
|
|
41
|
+
readonly message: string;
|
|
42
|
+
readonly guardrailId: GuardrailId;
|
|
43
|
+
readonly cause: unknown;
|
|
44
|
+
}
|
|
45
|
+
|
|
32
46
|
/**
|
|
33
47
|
* The declaration is invalid. From `defineGuardrail` /
|
|
34
48
|
* `validateGuardrailSpec`: it fails the schema, or (`defineGuardrail`)
|
|
@@ -148,6 +148,7 @@ export function makeLlmJudgeStrategy(
|
|
|
148
148
|
capability: Capability,
|
|
149
149
|
trace: RunTrace,
|
|
150
150
|
bindings: EvaluationBindings,
|
|
151
|
+
guardrailId?: string,
|
|
151
152
|
) => Promise<
|
|
152
153
|
CheckResult | { readonly error: { readonly code: string; readonly message: string } }
|
|
153
154
|
>,
|
|
@@ -171,6 +172,7 @@ export function makeLlmJudgeStrategy(
|
|
|
171
172
|
guardrail.judgeCapabilities,
|
|
172
173
|
trace,
|
|
173
174
|
bindings,
|
|
175
|
+
guardrail.id,
|
|
174
176
|
);
|
|
175
177
|
if ('error' in judged) {
|
|
176
178
|
return {
|
package/src/index.ts
CHANGED
package/src/judge.ts
CHANGED
|
@@ -1,10 +1,20 @@
|
|
|
1
1
|
// SPDX-License-Identifier: Apache-2.0
|
|
2
2
|
// Copyright (C) 2026 Kindgi Inc.
|
|
3
3
|
|
|
4
|
-
import {
|
|
5
|
-
import type { Capability, ModelInfo, ModelProvider } from '@kindgi/capabilities';
|
|
4
|
+
import { randomUUID } from 'node:crypto';
|
|
6
5
|
|
|
7
|
-
import
|
|
6
|
+
import { recordModelUsage, route } from '@kindgi/capabilities';
|
|
7
|
+
import type {
|
|
8
|
+
Capability,
|
|
9
|
+
ModelCallResult,
|
|
10
|
+
ModelInfo,
|
|
11
|
+
ModelProvider,
|
|
12
|
+
ModelUsageRecord,
|
|
13
|
+
} from '@kindgi/capabilities';
|
|
14
|
+
import { attemptsOf } from '@kindgi/capabilities/attempts';
|
|
15
|
+
import type { GuardrailId } from '@kindgi/types';
|
|
16
|
+
|
|
17
|
+
import type { JudgeMissingError, JudgeRoutingError, JudgeUsageError } from './errors.js';
|
|
8
18
|
import type { CheckResult, EvaluationBindings, RunTrace } from './types.js';
|
|
9
19
|
|
|
10
20
|
/**
|
|
@@ -30,14 +40,22 @@ export interface LlmJudgeConfig {
|
|
|
30
40
|
* Invoke an LLM judge to evaluate a guardrail. Resolves a model via
|
|
31
41
|
* the capability router (or uses `bindings.judgeProvider` if set), sends
|
|
32
42
|
* a structured judgment prompt including the run trace + rubric, parses
|
|
33
|
-
* the response.
|
|
43
|
+
* the response. The call is recorded in `bindings.usage` (the cost
|
|
44
|
+
* ledger) before its judgment is used, a call that threw included;
|
|
45
|
+
* `guardrailId` says which guardrail it judged. A sink that fails is
|
|
46
|
+
* tried again; an answered call it still can't record is a
|
|
47
|
+
* `judge-usage-unrecorded` error, not a judgment. A call that threw
|
|
48
|
+
* throws its own error, recorded or not: it was billed nothing.
|
|
34
49
|
*/
|
|
35
50
|
export async function invokeJudge(
|
|
36
51
|
config: LlmJudgeConfig,
|
|
37
52
|
capability: Capability,
|
|
38
53
|
trace: RunTrace,
|
|
39
54
|
bindings: EvaluationBindings,
|
|
40
|
-
|
|
55
|
+
guardrailId?: string,
|
|
56
|
+
): Promise<
|
|
57
|
+
CheckResult | { readonly error: JudgeMissingError | JudgeRoutingError | JudgeUsageError }
|
|
58
|
+
> {
|
|
41
59
|
const provider = await resolveJudgeProvider(capability, trace, bindings);
|
|
42
60
|
if ('error' in provider) return provider;
|
|
43
61
|
|
|
@@ -45,27 +63,98 @@ export async function invokeJudge(
|
|
|
45
63
|
const rubric = config.rubric;
|
|
46
64
|
const prompt = buildJudgePrompt(rubric, responseFormat, trace);
|
|
47
65
|
|
|
48
|
-
const
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
66
|
+
const record = (call: JudgeCallOutcome): Promise<unknown | undefined> =>
|
|
67
|
+
recordJudgeCall(bindings, trace, provider, guardrailId, call);
|
|
68
|
+
const callId = randomUUID();
|
|
69
|
+
const startedAt = Date.now();
|
|
70
|
+
let result: ModelCallResult;
|
|
71
|
+
try {
|
|
72
|
+
result = await provider.provider.invoke({
|
|
73
|
+
model: provider.model.name,
|
|
74
|
+
messages: [
|
|
75
|
+
{
|
|
76
|
+
role: 'system',
|
|
77
|
+
content:
|
|
78
|
+
responseFormat === 'pass-fail'
|
|
79
|
+
? 'You are an evaluator. Respond starting with PASS or FAIL on the first line, then a brief reason on the next line. No preamble.'
|
|
80
|
+
: 'You are an evaluator. Respond with a numeric score from 0 to 1 on the first line, then a brief reason on the next line. No preamble.',
|
|
81
|
+
},
|
|
82
|
+
{ role: 'user', content: prompt },
|
|
83
|
+
],
|
|
84
|
+
...(config.temperature !== undefined && { temperature: config.temperature }),
|
|
85
|
+
maxOutputTokens: 256,
|
|
86
|
+
...(bindings.abortSignal !== undefined && { abortSignal: bindings.abortSignal }),
|
|
87
|
+
});
|
|
88
|
+
} catch (cause) {
|
|
89
|
+
const attempts = attemptsOf(cause);
|
|
90
|
+
await record({
|
|
91
|
+
callId,
|
|
92
|
+
status: 'failed',
|
|
93
|
+
error: {
|
|
94
|
+
message: cause instanceof Error ? cause.message : String(cause),
|
|
95
|
+
...(attempts !== undefined && { attempts }),
|
|
57
96
|
},
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
97
|
+
durationMs: Date.now() - startedAt,
|
|
98
|
+
});
|
|
99
|
+
throw cause;
|
|
100
|
+
}
|
|
101
|
+
const { message: _answer, ...used } = result;
|
|
102
|
+
const unrecorded = await record({
|
|
103
|
+
callId,
|
|
104
|
+
status: 'ok',
|
|
105
|
+
result: used,
|
|
106
|
+
durationMs: result.durationMs,
|
|
63
107
|
});
|
|
108
|
+
if (unrecorded !== undefined) {
|
|
109
|
+
return {
|
|
110
|
+
error: {
|
|
111
|
+
code: 'judge-usage-unrecorded',
|
|
112
|
+
message: `The judge's model call couldn't be recorded: ${unrecorded instanceof Error ? unrecorded.message : String(unrecorded)}`,
|
|
113
|
+
guardrailId: (guardrailId ?? '<unknown>') as GuardrailId,
|
|
114
|
+
cause: unrecorded,
|
|
115
|
+
},
|
|
116
|
+
};
|
|
117
|
+
}
|
|
64
118
|
|
|
65
119
|
const text = result.message.content.trim();
|
|
66
120
|
return parseJudgeResponse(text, responseFormat, config.threshold);
|
|
67
121
|
}
|
|
68
122
|
|
|
123
|
+
/** What a judge call came to, for `recordJudgeCall`. */
|
|
124
|
+
type JudgeCallOutcome = Pick<
|
|
125
|
+
ModelUsageRecord,
|
|
126
|
+
'callId' | 'status' | 'result' | 'error' | 'durationMs'
|
|
127
|
+
>;
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Record a judge call in the usage sink, with the run's identity from the
|
|
131
|
+
* trace, trying a failing sink again. Resolves with the sink's last
|
|
132
|
+
* failure when it couldn't record.
|
|
133
|
+
*/
|
|
134
|
+
async function recordJudgeCall(
|
|
135
|
+
bindings: EvaluationBindings,
|
|
136
|
+
trace: RunTrace,
|
|
137
|
+
provider: ResolvedProvider,
|
|
138
|
+
guardrailId: string | undefined,
|
|
139
|
+
call: JudgeCallOutcome,
|
|
140
|
+
): Promise<unknown | undefined> {
|
|
141
|
+
if (bindings.usage === undefined) return;
|
|
142
|
+
const recorded = await recordModelUsage(bindings.usage, {
|
|
143
|
+
...call,
|
|
144
|
+
tenantId: trace.tenantId,
|
|
145
|
+
...(trace.projectId !== undefined && { projectId: trace.projectId }),
|
|
146
|
+
runId: trace.runId,
|
|
147
|
+
...(trace.agentId !== undefined && { agentId: trace.agentId as unknown as string }),
|
|
148
|
+
...(trace.conversationId !== undefined && { conversationId: trace.conversationId }),
|
|
149
|
+
providerId: provider.provider.metadata.id,
|
|
150
|
+
model: provider.model.name,
|
|
151
|
+
...(provider.provider.metadata.fallback === true && { fallback: true }),
|
|
152
|
+
purpose: guardrailId !== undefined ? `guardrail-judge:${guardrailId}` : 'guardrail-judge',
|
|
153
|
+
occurredAt: new Date().toISOString(),
|
|
154
|
+
});
|
|
155
|
+
return recorded.kind === 'err' ? (recorded.error ?? new Error('no detail')) : undefined;
|
|
156
|
+
}
|
|
157
|
+
|
|
69
158
|
interface ResolvedProvider {
|
|
70
159
|
readonly provider: ModelProvider;
|
|
71
160
|
readonly model: ModelInfo;
|
package/src/types.ts
CHANGED
|
@@ -6,12 +6,14 @@ import type {
|
|
|
6
6
|
ModelProvider,
|
|
7
7
|
ProviderRegistry,
|
|
8
8
|
TenantPolicy,
|
|
9
|
+
UsageSink,
|
|
9
10
|
} from '@kindgi/capabilities';
|
|
10
11
|
import type { ComplianceProvider } from '@kindgi/compliance';
|
|
11
12
|
import type {
|
|
12
13
|
AgentId,
|
|
13
14
|
FlowId,
|
|
14
15
|
GuardrailId,
|
|
16
|
+
OrgId,
|
|
15
17
|
ProjectId,
|
|
16
18
|
RunId,
|
|
17
19
|
TenantId,
|
|
@@ -307,6 +309,12 @@ export interface RunTrace {
|
|
|
307
309
|
* Optional; when absent, the violation emit is skipped.
|
|
308
310
|
*/
|
|
309
311
|
readonly projectId?: ProjectId;
|
|
312
|
+
/**
|
|
313
|
+
* The org of that project, when it belongs to one: resolved by the
|
|
314
|
+
* runtime from the project, never from input. Absent when the project
|
|
315
|
+
* has no org.
|
|
316
|
+
*/
|
|
317
|
+
readonly orgId?: OrgId;
|
|
310
318
|
readonly agentId?: AgentId;
|
|
311
319
|
readonly flowId?: FlowId;
|
|
312
320
|
/** Final assistant output text. Present when the run produced text. */
|
|
@@ -377,6 +385,12 @@ export interface EvaluationBindings {
|
|
|
377
385
|
* tests + hermetic pinning; production should route through the registry.
|
|
378
386
|
*/
|
|
379
387
|
readonly judgeProvider?: ModelProvider;
|
|
388
|
+
/**
|
|
389
|
+
* Where llm-judge guardrails record their model calls (the runtime's
|
|
390
|
+
* cost ledger), with the run's identity from the trace, before the
|
|
391
|
+
* judgment is used. Absent: judge calls aren't recorded.
|
|
392
|
+
*/
|
|
393
|
+
readonly usage?: UsageSink;
|
|
380
394
|
/**
|
|
381
395
|
* The tenant's model-routing policy (provider / model allow and deny
|
|
382
396
|
* lists, `regionAllow`, caps). When present, llm-judge models are
|