@kindgi/guardrails 0.1.4-rc.4 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/judge.d.ts +8 -0
- package/dist/judge.d.ts.map +1 -1
- package/dist/judge.js +12 -1
- package/dist/judge.js.map +1 -1
- package/package.json +6 -6
- package/src/index.ts +1 -1
- package/src/judge.ts +14 -1
package/README.md
CHANGED
|
@@ -70,6 +70,10 @@ Same trade-off as `@kindgi/tools`: JSON Schema for portable / MCP-published chec
|
|
|
70
70
|
|
|
71
71
|
Note that `Guardrail.config` on the wire is always `object` with `additionalProperties: true` — the check declares the concrete schema. Authors extend the guardrail registration surface (`defineCheck`); the guardrail surface itself takes an opaque `config` payload that the resolved check validates when `defineGuardrail` runs.
|
|
72
72
|
|
|
73
|
+
## The `llm-judge` strategy
|
|
74
|
+
|
|
75
|
+
A check of kind `llm-judge` asks a registered model for a verdict on the run trace: PASS or FAIL, or a score from 0 to 1, with a brief reason. Its answer gets 256 tokens. On a model that thinks (its registration has `thinking`), the judge asks for the least thinking the model allows and adds 2,048 tokens for it, since thinking counts against the output limit. On a model that takes no temperature (`"sampling": false`), a judge's `temperature` is left out. Each judge call is recorded in the cost ledger, under the guardrail it judged.
|
|
76
|
+
|
|
73
77
|
## Peer dependency
|
|
74
78
|
|
|
75
79
|
`zod` (>= 4.0.0) is a peer dependency. Workspaces that never author with Zod never install it. Zod schemas that fail to convert (unrepresentable constructs like `z.function()`) surface as `invalid-check-definition` thrown at pack init — same failure model as an uncompilable JSON Schema.
|
package/dist/index.d.ts
CHANGED
|
@@ -8,7 +8,7 @@ export { builtInStrategies, evaluateAll, evaluateGuardrail, violations } from '.
|
|
|
8
8
|
export type { EvaluationOutcome } from './engine.js';
|
|
9
9
|
export { createExecutionStrategyRegistry, externalStrategy, makeLlmJudgeStrategy, zeroLlmStrategy, } from './execution-strategy.js';
|
|
10
10
|
export type { ExecutionStrategy, ExecutionStrategyRegistry, StrategyError, StrategyResult, } from './execution-strategy.js';
|
|
11
|
-
export { invokeJudge } from './judge.js';
|
|
11
|
+
export { JUDGE_THINKING_TOKENS, JUDGE_VERDICT_TOKENS, invokeJudge } from './judge.js';
|
|
12
12
|
export type { LlmJudgeConfig } from './judge.js';
|
|
13
13
|
export type { Action, Budget, BuiltInGuardrailKind, BuiltInOnViolation, CheckFunction, CheckRegistry, CheckResult, CodeArtifactRef, EvaluationBindings, EvaluationResult, Guardrail, GuardrailKind, GuardrailSeverity, JsonSchema, ModelCallRecord, NetworkPolicy, OnViolation, RegisteredCheck, RunTrace, RuntimeLimits, SandboxMode, Scope, ScopeWhen, ToolCallRecord, ToolResultRecord, TypedNeeds, } from './types.js';
|
|
14
14
|
export { BUILT_IN_GUARDRAIL_KINDS, BUILT_IN_ON_VIOLATIONS } from './types.js';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAC3F,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EAAE,eAAe,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACzF,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,2BAA2B,EAC3B,eAAe,EACf,WAAW,EACX,cAAc,EACd,WAAW,EACX,YAAY,GACb,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EACV,aAAa,EACb,aAAa,EACb,qBAAqB,EACrB,YAAY,GACb,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC5F,YAAY,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EACL,+BAA+B,EAC/B,gBAAgB,EAChB,oBAAoB,EACpB,eAAe,GAChB,MAAM,yBAAyB,CAAC;AACjC,YAAY,EACV,iBAAiB,EACjB,yBAAyB,EACzB,aAAa,EACb,cAAc,GACf,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAC3F,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EAAE,eAAe,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACzF,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,2BAA2B,EAC3B,eAAe,EACf,WAAW,EACX,cAAc,EACd,WAAW,EACX,YAAY,GACb,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EACV,aAAa,EACb,aAAa,EACb,qBAAqB,EACrB,YAAY,GACb,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC5F,YAAY,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EACL,+BAA+B,EAC/B,gBAAgB,EAChB,oBAAoB,EACpB,eAAe,GAChB,MAAM,yBAAyB,CAAC;AACjC,YAAY,EACV,iBAAiB,EACjB,yBAAyB,EACzB,aAAa,EACb,cAAc,GACf,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACtF,YAAY,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,YAAY,EACV,MAAM,EACN,MAAM,EACN,oBAAoB,EACpB,kBAAkB,EAClB,aAAa,EACb,aAAa,EACb,WAAW,EACX,eAAe,EACf,kBAAkB,EAClB,gBAAgB,EAChB,SAAS,EACT,aAAa,EACb,iBAAiB,EACjB,UAAU,EACV,eAAe,EACf,aAAa,EACb,WAAW,EACX,eAAe,EACf,QAAQ,EACR,aAAa,EACb,WAAW,EACX,KAAK,EACL,SAAS,EACT,cAAc,EACd,gBAAgB,EAChB,UAAU,GACX,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAC9E,YAAY,EACV,uBAAuB,EACvB,2BAA2B,EAC3B,qBAAqB,EACrB,cAAc,EACd,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,kBAAkB,EAClB,kBAAkB,EAClB,iBAAiB,GAClB,MAAM,aAAa,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -6,6 +6,6 @@ export { BUILT_IN_CHECK_IDS, createCheckRegistry } from './checks.js';
|
|
|
6
6
|
export { BUILT_IN_ACTION_HANDLERS, compensateHandler, createActionHandlerRegistry, escalateHandler, haltHandler, logOnlyHandler, noopHandler, retryHandler, } from './action-handler.js';
|
|
7
7
|
export { builtInStrategies, evaluateAll, evaluateGuardrail, violations } from './engine.js';
|
|
8
8
|
export { createExecutionStrategyRegistry, externalStrategy, makeLlmJudgeStrategy, zeroLlmStrategy, } from './execution-strategy.js';
|
|
9
|
-
export { invokeJudge } from './judge.js';
|
|
9
|
+
export { JUDGE_THINKING_TOKENS, JUDGE_VERDICT_TOKENS, invokeJudge } from './judge.js';
|
|
10
10
|
export { BUILT_IN_GUARDRAIL_KINDS, BUILT_IN_ON_VIOLATIONS } from './types.js';
|
|
11
11
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,iCAAiC;AAEjC,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAC3F,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,2BAA2B,EAC3B,eAAe,EACf,WAAW,EACX,cAAc,EACd,WAAW,EACX,YAAY,GACb,MAAM,qBAAqB,CAAC;AAO7B,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE5F,OAAO,EACL,+BAA+B,EAC/B,gBAAgB,EAChB,oBAAoB,EACpB,eAAe,GAChB,MAAM,yBAAyB,CAAC;AAOjC,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,iCAAiC;AAEjC,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAC3F,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,2BAA2B,EAC3B,eAAe,EACf,WAAW,EACX,cAAc,EACd,WAAW,EACX,YAAY,GACb,MAAM,qBAAqB,CAAC;AAO7B,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE5F,OAAO,EACL,+BAA+B,EAC/B,gBAAgB,EAChB,oBAAoB,EACpB,eAAe,GAChB,MAAM,yBAAyB,CAAC;AAOjC,OAAO,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AA8BtF,OAAO,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC"}
|
package/dist/judge.d.ts
CHANGED
|
@@ -19,6 +19,14 @@ export interface LlmJudgeConfig {
|
|
|
19
19
|
/** Optional override for the model's temperature. */
|
|
20
20
|
readonly temperature?: number;
|
|
21
21
|
}
|
|
22
|
+
/** A verdict's budget: PASS/FAIL or a score, and a brief reason. */
|
|
23
|
+
export declare const JUDGE_VERDICT_TOKENS = 256;
|
|
24
|
+
/**
|
|
25
|
+
* Added for a model that thinks (`ModelInfo.thinking`): even at its least
|
|
26
|
+
* thinking (the judge asks for `thinking: 'lowest'`), thought tokens count
|
|
27
|
+
* against the cap, and a verdict mustn't be crowded out.
|
|
28
|
+
*/
|
|
29
|
+
export declare const JUDGE_THINKING_TOKENS = 2048;
|
|
22
30
|
/**
|
|
23
31
|
* Invoke an LLM judge to evaluate a guardrail. Resolves a model via
|
|
24
32
|
* the capability router (or uses `bindings.judgeProvider` if set), sends
|
package/dist/judge.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../src/judge.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EACV,UAAU,EAKX,MAAM,sBAAsB,CAAC;AAI9B,OAAO,KAAK,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACzF,OAAO,KAAK,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE5E;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,kEAAkE;IAClE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;;;;OAIG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC;IAChD,wEAAwE;IACxE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,qDAAqD;IACrD,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,WAAW,CAC/B,MAAM,EAAE,cAAc,EACtB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,QAAQ,EACf,QAAQ,EAAE,kBAAkB,EAC5B,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CACR,WAAW,GAAG;IAAE,QAAQ,CAAC,KAAK,EAAE,iBAAiB,GAAG,iBAAiB,GAAG,eAAe,CAAA;CAAE,CAC1F,
|
|
1
|
+
{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../src/judge.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EACV,UAAU,EAKX,MAAM,sBAAsB,CAAC;AAI9B,OAAO,KAAK,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACzF,OAAO,KAAK,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE5E;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,kEAAkE;IAClE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;;;;OAIG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC;IAChD,wEAAwE;IACxE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,qDAAqD;IACrD,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,oEAAoE;AACpE,eAAO,MAAM,oBAAoB,MAAM,CAAC;AACxC;;;;GAIG;AACH,eAAO,MAAM,qBAAqB,OAAO,CAAC;AAE1C;;;;;;;;;;GAUG;AACH,wBAAsB,WAAW,CAC/B,MAAM,EAAE,cAAc,EACtB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,QAAQ,EACf,QAAQ,EAAE,kBAAkB,EAC5B,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CACR,WAAW,GAAG;IAAE,QAAQ,CAAC,KAAK,EAAE,iBAAiB,GAAG,iBAAiB,GAAG,eAAe,CAAA;CAAE,CAC1F,CAmEA"}
|
package/dist/judge.js
CHANGED
|
@@ -3,6 +3,14 @@
|
|
|
3
3
|
import { randomUUID } from 'node:crypto';
|
|
4
4
|
import { recordModelUsage, route } from '@kindgi/capabilities';
|
|
5
5
|
import { attemptsOf } from '@kindgi/capabilities/attempts';
|
|
6
|
+
/** A verdict's budget: PASS/FAIL or a score, and a brief reason. */
|
|
7
|
+
export const JUDGE_VERDICT_TOKENS = 256;
|
|
8
|
+
/**
|
|
9
|
+
* Added for a model that thinks (`ModelInfo.thinking`): even at its least
|
|
10
|
+
* thinking (the judge asks for `thinking: 'lowest'`), thought tokens count
|
|
11
|
+
* against the cap, and a verdict mustn't be crowded out.
|
|
12
|
+
*/
|
|
13
|
+
export const JUDGE_THINKING_TOKENS = 2048;
|
|
6
14
|
/**
|
|
7
15
|
* Invoke an LLM judge to evaluate a guardrail. Resolves a model via
|
|
8
16
|
* the capability router (or uses `bindings.judgeProvider` if set), sends
|
|
@@ -38,7 +46,10 @@ export async function invokeJudge(config, capability, trace, bindings, guardrail
|
|
|
38
46
|
{ role: 'user', content: prompt },
|
|
39
47
|
],
|
|
40
48
|
...(config.temperature !== undefined && { temperature: config.temperature }),
|
|
41
|
-
|
|
49
|
+
...(provider.model.thinking !== undefined && { thinking: 'lowest' }),
|
|
50
|
+
maxOutputTokens: provider.model.thinking !== undefined
|
|
51
|
+
? JUDGE_VERDICT_TOKENS + JUDGE_THINKING_TOKENS
|
|
52
|
+
: JUDGE_VERDICT_TOKENS,
|
|
42
53
|
...(bindings.abortSignal !== undefined && { abortSignal: bindings.abortSignal }),
|
|
43
54
|
});
|
|
44
55
|
}
|
package/dist/judge.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"judge.js","sourceRoot":"","sources":["../src/judge.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,iCAAiC;AAEjC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,gBAAgB,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAQ/D,OAAO,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAyB3D;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAsB,EACtB,UAAsB,EACtB,KAAe,EACf,QAA4B,EAC5B,WAAoB;IAIpB,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,UAAU,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;IACzE,IAAI,OAAO,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAEzC,MAAM,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,WAAW,CAAC;IAC5D,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7B,MAAM,MAAM,GAAG,gBAAgB,CAAC,MAAM,EAAE,cAAc,EAAE,KAAK,CAAC,CAAC;IAE/D,MAAM,MAAM,GAAG,CAAC,IAAsB,EAAgC,EAAE,CACtE,eAAe,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC;IAChE,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,IAAI,MAAuB,CAAC;IAC5B,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;YACtC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI;YAC1B,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,QAAQ;oBACd,OAAO,EACL,cAAc,KAAK,WAAW;wBAC5B,CAAC,CAAC,gIAAgI;wBAClI,CAAC,CAAC,sIAAsI;iBAC7I;gBACD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE;aAClC;YACD,GAAG,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS,IAAI,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC;YAC5E,
|
|
1
|
+
{"version":3,"file":"judge.js","sourceRoot":"","sources":["../src/judge.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,iCAAiC;AAEjC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,gBAAgB,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAQ/D,OAAO,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAyB3D,oEAAoE;AACpE,MAAM,CAAC,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACxC;;;;GAIG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,CAAC;AAE1C;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAsB,EACtB,UAAsB,EACtB,KAAe,EACf,QAA4B,EAC5B,WAAoB;IAIpB,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,UAAU,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;IACzE,IAAI,OAAO,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAEzC,MAAM,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,WAAW,CAAC;IAC5D,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7B,MAAM,MAAM,GAAG,gBAAgB,CAAC,MAAM,EAAE,cAAc,EAAE,KAAK,CAAC,CAAC;IAE/D,MAAM,MAAM,GAAG,CAAC,IAAsB,EAAgC,EAAE,CACtE,eAAe,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC;IAChE,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,IAAI,MAAuB,CAAC;IAC5B,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;YACtC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI;YAC1B,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,QAAQ;oBACd,OAAO,EACL,cAAc,KAAK,WAAW;wBAC5B,CAAC,CAAC,gIAAgI;wBAClI,CAAC,CAAC,sIAAsI;iBAC7I;gBACD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE;aAClC;YACD,GAAG,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS,IAAI,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC;YAC5E,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,QAAiB,EAAE,CAAC;YAC7E,eAAe,EACb,QAAQ,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS;gBACnC,CAAC,CAAC,oBAAoB,GAAG,qBAAqB;gBAC9C,CAAC,CAAC,oBAAoB;YAC1B,GAAG,CAAC,QAAQ,CAAC,WAAW,KAAK,SAAS,IAAI,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC;SACjF,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QACnC,MAAM,MAAM,CAAC;YACX,MAAM;YACN,MAAM,EAAE,QAAQ;YAChB,KAAK,EAAE;gBACL,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;gBAC/D,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,CAAC;aAC5C;YACD,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SACnC,CAAC,CAAC;QACH,MAAM,KAAK,CAAC;IACd,CAAC;IACD,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC;IAC7C,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC;QAC9B,MAAM;QACN,MAAM,EAAE,IAAI;QACZ,MAAM,EAAE,IAAI;QACZ,UAAU,EAAE,MAAM,CAAC,UAAU;KAC9B,CAAC,CAAC;IACH,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO;YACL,KAAK,EAAE;gBACL,IAAI,EAAE,wBAAwB;gBAC9B,OAAO,EAAE,gDAAgD,UAAU,YAAY,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE;gBAChI,WAAW,EAAE,CAAC,WAAW,IAAI,WAAW,CAAgB;gBACxD,KAAK,EAAE,UAAU;aAClB;SACF,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IAC3C,OAAO,kBAAkB,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;AACpE,CAAC;AAQD;;;;GAIG;AACH,KAAK,UAAU,eAAe,CAC5B,QAA4B,EAC5B,KAAe,EACf,QAA0B,EAC1B,WAA+B,EAC/B,IAAsB;IAEtB,IAAI,QAAQ,CAAC,KAAK,KAAK,SAAS;QAAE,OAAO;IACzC,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,QAAQ,CAAC,KAAK,EAAE;QACtD,GAAG,IAAI;QACP,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,GAAG,CAAC,KAAK,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC;QACpE,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,OAA4B,EAAE,CAAC;QACnF,GAAG,CAAC,KAAK,CAAC,cAAc,KAAK,SAAS,IAAI,EAAE,cAAc,EAAE,KAAK,CAAC,cAAc,EAAE,CAAC;QACnF,UAAU,EAAE,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;QACzC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI;QAC1B,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,KAAK,IAAI,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QACvE,OAAO,EAAE,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,mBAAmB,WAAW,EAAE,CAAC,CAAC,CAAC,iBAAiB;QACzF,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACrC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,IAAI,IAAI,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC1F,CAAC;AAQD,KAAK,UAAU,oBAAoB,CACjC,UAAsB,EACtB,KAAe,EACf,QAA4B;IAE5B,IAAI,QAAQ,CAAC,aAAa,KAAK,SAAS,IAAI,QAAQ,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QAChF,oEAAoE;QACpE,qDAAqD;QACrD,MAAM,QAAQ,GAAG,KAAK,CAAC;YACrB,UAAU;YACV,SAAS,EAAE,CAAC,QAAQ,CAAC,aAAa,CAAC;YACnC,YAAY,EAAE,QAAQ,CAAC,YAAY;SACpC,CAAC,CAAC;QACH,IAAI,QAAQ,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAsB;gBAC7B,IAAI,EAAE,sBAAsB;gBAC5B,OAAO,EAAE,2BAA2B,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,qCAAqC,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE;gBACnI,WAAW,EAAE,WAAoB;gBACjC,KAAK,EAAE,QAAQ,CAAC,KAAK;aACtB,CAAC;YACF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;QACxB,CAAC;QACD,OAAO;YACL,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,QAAQ;YACjC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,KAAK;YAC3B,MAAM,EAAE,oCAAoC,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE;SACpE,CAAC;IACJ,CAAC;IACD,IAAI,QAAQ,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QACzC,gEAAgE;QAChE,mEAAmE;QACnE,kEAAkE;QAClE,mCAAmC;QACnC,MAAM,QAAQ,GAAG,QAAQ,CAAC,aAAa,CAAC;QACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,GAAG,GAAsB;gBAC7B,IAAI,EAAE,sBAAsB;gBAC5B,OAAO,EAAE,2BAA2B,QAAQ,CAAC,QAAQ,CAAC,EAAE,uBAAuB;gBAC/E,WAAW,EAAE,WAAoB;gBACjC,KAAK,EAAE;oBACL,IAAI,EAAE,0BAA0B;oBAChC,OAAO,EAAE,6BAA6B;oBACtC,OAAO,EAAE,EAAE;iBACZ;aACF,CAAC;YACF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;QACxB,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,iCAAiC,EAAE,CAAC;IAC/E,CAAC;IACD,IAAI,QAAQ,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;QAC5C,MAAM,GAAG,GAAsB;YAC7B,IAAI,EAAE,eAAe;YACrB,OAAO,EACL,qFAAqF;YACvF,WAAW,EAAE,WAAoB;SAClC,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IACxB,CAAC;IACD,MAAM,QAAQ,GAAG,KAAK,CAAC;QACrB,UAAU;QACV,SAAS,EAAE,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;QACzD,GAAG,CAAC,QAAQ,CAAC,YAAY,KAAK,SAAS,IAAI,EAAE,YAAY,EAAE,QAAQ,CAAC,YAAY,EAAE,CAAC;KACpF,CAAC,CAAC;IACH,IAAI,QAAQ,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAsB;YAC7B,IAAI,EAAE,sBAAsB;YAC5B,OAAO,EAAE,yBAAyB,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE;YAC1D,WAAW,EAAE,WAAoB;YACjC,KAAK,EAAE,QAAQ,CAAC,KAAK;SACtB,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IACxB,CAAC;IACD,OAAO;QACL,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,QAAQ;QACjC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,KAAK;QAC3B,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM;KAC9B,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAc,EAAE,MAA6B,EAAE,KAAe;IACtF,MAAM,KAAK,GAAG;QACZ,sDAAsD;QACtD,EAAE;QACF,gBAAgB;QAChB,MAAM;QACN,EAAE;QACF,oBAAoB;QACpB,KAAK,CAAC,MAAM,IAAI,SAAS;QACzB,EAAE;QACF,oBAAoB;QACpB,KAAK,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;YAC1B,CAAC,CAAC,QAAQ;YACV,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,QAAQ,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;QAC5F,EAAE;QACF,MAAM,KAAK,WAAW;YACpB,CAAC,CAAC,mCAAmC;YACrC,CAAC,CAAC,6CAA6C;KAClD,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,kBAAkB,CACzB,GAAW,EACX,MAA6B,EAC7B,SAA6B;IAE7B,IAAI,MAAM,KAAK,WAAW,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QAChC,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC7B,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YACvD,MAAM,MAAM,GAAgB,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,CAAC;YACjE,OAAO,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QAC5E,CAAC;QACD,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC7B,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YACvD,OAAO;gBACL,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,qBAAqB;gBAClE,aAAa,EAAE,GAAG;aACnB,CAAC;QACJ,CAAC;QACD,OAAO;YACL,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,gDAAgD,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE;YAC1E,aAAa,EAAE,GAAG;SACnB,CAAC;IACJ,CAAC;IACD,eAAe;IACf,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACtD,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IAC3C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO;YACL,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,yCAAyC,SAAS,EAAE;YAC5D,aAAa,EAAE,GAAG;SACnB,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,SAAS,IAAI,GAAG,CAAC;IAChC,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACvD,OAAO;QACL,MAAM,EAAE,KAAK,IAAI,MAAM;QACvB,MAAM,EAAE,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,KAAK,eAAe,MAAM,EAAE;QAClF,aAAa,EAAE,GAAG;QAClB,UAAU,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE;KACzC,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kindgi/guardrails",
|
|
3
|
-
"version": "0.1.4
|
|
3
|
+
"version": "0.1.4",
|
|
4
4
|
"description": "Runtime + CI enforcement of agent-behavior guardrails for Kindgi. Same declaration runs at runtime (halt / retry / escalate / log-only / compensate on violation) and in CI (fails the build) — no drift between test and prod. Zero-LLM checks (must-cite, never-call-tool, max-tool-calls, output-matches, tool-order, required-substring, forbidden-substring) are the default fast path; LLM-judge guardrails route via @kindgi/capabilities with explicit budget declarations. Execution strategies and action handlers are pluggable registries, and failed checks optionally emit compliance evidence.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -30,10 +30,10 @@
|
|
|
30
30
|
"type-manifest.json"
|
|
31
31
|
],
|
|
32
32
|
"dependencies": {
|
|
33
|
-
"@kindgi/capabilities": "0.1.4
|
|
34
|
-
"@kindgi/compliance": "0.1.4
|
|
35
|
-
"@kindgi/schema": "0.1.4
|
|
36
|
-
"@kindgi/types": "0.1.4
|
|
33
|
+
"@kindgi/capabilities": "0.1.4",
|
|
34
|
+
"@kindgi/compliance": "0.1.4",
|
|
35
|
+
"@kindgi/schema": "0.1.4",
|
|
36
|
+
"@kindgi/types": "0.1.4",
|
|
37
37
|
"ajv": "^8.17.1",
|
|
38
38
|
"ajv-formats": "^3.0.1"
|
|
39
39
|
},
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
}
|
|
47
47
|
},
|
|
48
48
|
"devDependencies": {
|
|
49
|
-
"@kindgi/specs": "0.1.4
|
|
49
|
+
"@kindgi/specs": "0.1.4",
|
|
50
50
|
"@types/node": "^22.10.5",
|
|
51
51
|
"typescript": "^5.7.3",
|
|
52
52
|
"vitest": "^2.1.8",
|
package/src/index.ts
CHANGED
|
@@ -35,7 +35,7 @@ export type {
|
|
|
35
35
|
StrategyError,
|
|
36
36
|
StrategyResult,
|
|
37
37
|
} from './execution-strategy.js';
|
|
38
|
-
export { invokeJudge } from './judge.js';
|
|
38
|
+
export { JUDGE_THINKING_TOKENS, JUDGE_VERDICT_TOKENS, invokeJudge } from './judge.js';
|
|
39
39
|
export type { LlmJudgeConfig } from './judge.js';
|
|
40
40
|
export type {
|
|
41
41
|
Action,
|
package/src/judge.ts
CHANGED
|
@@ -36,6 +36,15 @@ export interface LlmJudgeConfig {
|
|
|
36
36
|
readonly temperature?: number;
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
+
/** A verdict's budget: PASS/FAIL or a score, and a brief reason. */
|
|
40
|
+
export const JUDGE_VERDICT_TOKENS = 256;
|
|
41
|
+
/**
|
|
42
|
+
* Added for a model that thinks (`ModelInfo.thinking`): even at its least
|
|
43
|
+
* thinking (the judge asks for `thinking: 'lowest'`), thought tokens count
|
|
44
|
+
* against the cap, and a verdict mustn't be crowded out.
|
|
45
|
+
*/
|
|
46
|
+
export const JUDGE_THINKING_TOKENS = 2048;
|
|
47
|
+
|
|
39
48
|
/**
|
|
40
49
|
* Invoke an LLM judge to evaluate a guardrail. Resolves a model via
|
|
41
50
|
* the capability router (or uses `bindings.judgeProvider` if set), sends
|
|
@@ -82,7 +91,11 @@ export async function invokeJudge(
|
|
|
82
91
|
{ role: 'user', content: prompt },
|
|
83
92
|
],
|
|
84
93
|
...(config.temperature !== undefined && { temperature: config.temperature }),
|
|
85
|
-
|
|
94
|
+
...(provider.model.thinking !== undefined && { thinking: 'lowest' as const }),
|
|
95
|
+
maxOutputTokens:
|
|
96
|
+
provider.model.thinking !== undefined
|
|
97
|
+
? JUDGE_VERDICT_TOKENS + JUDGE_THINKING_TOKENS
|
|
98
|
+
: JUDGE_VERDICT_TOKENS,
|
|
86
99
|
...(bindings.abortSignal !== undefined && { abortSignal: bindings.abortSignal }),
|
|
87
100
|
});
|
|
88
101
|
} catch (cause) {
|