@framers/agentos 0.9.70 → 0.9.72
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/safety/LLMProviderHealthRegistry.d.ts.map +1 -1
- package/dist/core/safety/LLMProviderHealthRegistry.js +19 -0
- package/dist/core/safety/LLMProviderHealthRegistry.js.map +1 -1
- package/dist/safety/evaluation/LLMJudge.d.ts +8 -0
- package/dist/safety/evaluation/LLMJudge.d.ts.map +1 -1
- package/dist/safety/evaluation/LLMJudge.js +5 -2
- package/dist/safety/evaluation/LLMJudge.js.map +1 -1
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LLMProviderHealthRegistry.d.ts","sourceRoot":"","sources":["../../../src/core/safety/LLMProviderHealthRegistry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgFG;AAEH,uDAAuD;AACvD,MAAM,WAAW,sBAAsB;IACrC,2CAA2C;IAC3C,UAAU,EAAE,MAAM,CAAC;IACnB,6CAA6C;IAC7C,KAAK,EAAE,QAAQ,GAAG,MAAM,CAAC;IACzB,0EAA0E;IAC1E,mBAAmB,EAAE,MAAM,CAAC;IAC5B,+DAA+D;IAC/D,YAAY,EAAE,MAAM,CAAC;IACrB,+EAA+E;IAC/E,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,wDAAwD;IACxD,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,4DAA4D;IAC5D,UAAU,EAAE,MAAM,CAAC;CACpB;AA2BD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAoBjE;
|
|
1
|
+
{"version":3,"file":"LLMProviderHealthRegistry.d.ts","sourceRoot":"","sources":["../../../src/core/safety/LLMProviderHealthRegistry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgFG;AAEH,uDAAuD;AACvD,MAAM,WAAW,sBAAsB;IACrC,2CAA2C;IAC3C,UAAU,EAAE,MAAM,CAAC;IACnB,6CAA6C;IAC7C,KAAK,EAAE,QAAQ,GAAG,MAAM,CAAC;IACzB,0EAA0E;IAC1E,mBAAmB,EAAE,MAAM,CAAC;IAC5B,+DAA+D;IAC/D,YAAY,EAAE,MAAM,CAAC;IACrB,+EAA+E;IAC/E,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,wDAAwD;IACxD,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,4DAA4D;IAC5D,UAAU,EAAE,MAAM,CAAC;CACpB;AA2BD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAoBjE;AAwBD;;;;;;;GAOG;AACH,qBAAa,yBAAyB;IACpC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAgD;IAExE;;;;;OAKG;IACH,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO;IAcnC;;;;;;;;;;OAUG;IACH,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,IAAI;IAsBvD;;;;;OAKG;IACH,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAMvC;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI;IAQhC;;;;OAIG;IACH,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,sBAAsB,GAAG,IAAI;IAiB3D,OAAO,CAAC,UAAU;CASnB;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,uBAAuB,EAAE,yBAA2D,CAAC"}
|
|
@@ -131,6 +131,20 @@ function policyForStatus(status) {
|
|
|
131
131
|
// 5xx + null (unclassifiable) all fall through to transient.
|
|
132
132
|
return POLICY_TRANSIENT;
|
|
133
133
|
}
|
|
134
|
+
/**
|
|
135
|
+
* A 4xx that reflects a bad request rather than provider health — the provider
|
|
136
|
+
* is up and answering; the CALLER's request was wrong (malformed body, unknown
|
|
137
|
+
* model, unprocessable params). These must not count toward the breaker, or a
|
|
138
|
+
* caller bug would disable a healthy provider (and would inflate the streak a
|
|
139
|
+
* later transient 5xx then trips on). Account-level (401/402/403), request
|
|
140
|
+
* timeout (408), and rate-limit (429) are deliberately excluded — those DO
|
|
141
|
+
* warrant failover and keep their own policies.
|
|
142
|
+
*/
|
|
143
|
+
function isNonHealthClientError(status) {
|
|
144
|
+
if (status === null || status < 400 || status >= 500)
|
|
145
|
+
return false;
|
|
146
|
+
return status !== 401 && status !== 402 && status !== 403 && status !== 408 && status !== 429;
|
|
147
|
+
}
|
|
134
148
|
/**
|
|
135
149
|
* Per-process registry of provider health. Construct once per agentos
|
|
136
150
|
* runtime; the module-level `globalLLMProviderHealth` singleton is
|
|
@@ -177,6 +191,11 @@ export class LLMProviderHealthRegistry {
|
|
|
177
191
|
*/
|
|
178
192
|
recordFailure(providerId, error) {
|
|
179
193
|
const status = classifyErrorStatus(error);
|
|
194
|
+
// A client-error 4xx (bad request / unknown model / unprocessable) is the
|
|
195
|
+
// caller's fault, not a provider-health signal — ignore it entirely so it
|
|
196
|
+
// neither trips the breaker nor inflates the streak.
|
|
197
|
+
if (isNonHealthClientError(status))
|
|
198
|
+
return;
|
|
180
199
|
const policy = policyForStatus(status);
|
|
181
200
|
const record = this.records.get(providerId) ?? this.makeRecord();
|
|
182
201
|
record.failureCount += 1;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LLMProviderHealthRegistry.js","sourceRoot":"","sources":["../../../src/core/safety/LLMProviderHealthRegistry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgFG;AAwCH,MAAM,UAAU,GAAgB,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,GAAG,KAAM,EAAE,CAAC;AACzE,MAAM,WAAW,GAAgB,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,EAAE,GAAG,KAAM,EAAE,CAAC;AAC3E,MAAM,UAAU,GAAgB,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,KAAM,EAAE,CAAC;AACrE,MAAM,gBAAgB,GAAgB,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,KAAM,EAAE,CAAC;AAE3E;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAc;IAChD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IACrD,MAAM,GAAG,GAAG,KAAsE,CAAC;IACnF,uBAAuB;IACvB,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAChD,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAE,EAAE,EAAE,CAAC,CAAC;YACrC,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,GAAG,IAAI,IAAI,GAAG,GAAG;gBAAE,OAAO,IAAI,CAAC;QACtE,CAAC;IACH,CAAC;IACD,yBAAyB;IACzB,IAAI,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,IAAI,GAAG,CAAC,UAAU,IAAI,GAAG,IAAI,GAAG,CAAC,UAAU,GAAG,GAAG,EAAE,CAAC;QACxF,OAAO,GAAG,CAAC,UAAU,CAAC;IACxB,CAAC;IACD,qBAAqB;IACrB,IAAI,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QAC5E,OAAO,GAAG,CAAC,MAAM,CAAC;IACpB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,eAAe,CAAC,MAAqB;IAC5C,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,UAAU,CAAC;IACtC,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,WAAW,CAAC;IACzD,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,UAAU,CAAC;IACtC,6DAA6D;IAC7D,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,OAAO,yBAAyB;IAAtC;QACmB,YAAO,GAAsC,IAAI,GAAG,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"LLMProviderHealthRegistry.js","sourceRoot":"","sources":["../../../src/core/safety/LLMProviderHealthRegistry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgFG;AAwCH,MAAM,UAAU,GAAgB,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,GAAG,KAAM,EAAE,CAAC;AACzE,MAAM,WAAW,GAAgB,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,EAAE,GAAG,KAAM,EAAE,CAAC;AAC3E,MAAM,UAAU,GAAgB,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,KAAM,EAAE,CAAC;AACrE,MAAM,gBAAgB,GAAgB,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,KAAM,EAAE,CAAC;AAE3E;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAc;IAChD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IACrD,MAAM,GAAG,GAAG,KAAsE,CAAC;IACnF,uBAAuB;IACvB,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAChD,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAE,EAAE,EAAE,CAAC,CAAC;YACrC,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,GAAG,IAAI,IAAI,GAAG,GAAG;gBAAE,OAAO,IAAI,CAAC;QACtE,CAAC;IACH,CAAC;IACD,yBAAyB;IACzB,IAAI,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,IAAI,GAAG,CAAC,UAAU,IAAI,GAAG,IAAI,GAAG,CAAC,UAAU,GAAG,GAAG,EAAE,CAAC;QACxF,OAAO,GAAG,CAAC,UAAU,CAAC;IACxB,CAAC;IACD,qBAAqB;IACrB,IAAI,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QAC5E,OAAO,GAAG,CAAC,MAAM,CAAC;IACpB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,eAAe,CAAC,MAAqB;IAC5C,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,UAAU,CAAC;IACtC,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,WAAW,CAAC;IACzD,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,UAAU,CAAC;IACtC,6DAA6D;IAC7D,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,sBAAsB,CAAC,MAAqB;IACnD,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG;QAAE,OAAO,KAAK,CAAC;IACnE,OAAO,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,CAAC;AAChG,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,OAAO,yBAAyB;IAAtC;QACmB,YAAO,GAAsC,IAAI,GAAG,EAAE,CAAC;IAgH1E,CAAC;IA9GC;;;;;OAKG;IACH,MAAM,CAAC,UAAkB;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC;QAC1B,IAAI,MAAM,CAAC,SAAS,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACzC,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACnC,8DAA8D;YAC9D,mEAAmE;YACnE,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC;YACrB,MAAM,CAAC,YAAY,GAAG,CAAC,CAAC;YACxB,OAAO,KAAK,CAAC;QACf,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;;;;;;OAUG;IACH,aAAa,CAAC,UAAkB,EAAE,KAAc;QAC9C,MAAM,MAAM,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;QAC1C,0EAA0E;QAC1E,0EAA0E;QAC1E,qDAAqD;QACrD,IAAI,sBAAsB,CAAC,MAAM,CAAC;YAAE,OAAO;QAC3C,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;QACvC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;QACjE,MAAM,CAAC,YAAY,IAAI,CAAC,CAAC;QACzB,MAAM,CAAC,cAAc,GAAG,MAAM,CAAC;QAC/B,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAClC,IAAI,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YAC5C,uDAAuD;YACvD,iCAAiC;YACjC,IAAI,MAAM,CAAC,SAAS,KAAK,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;gBAC7D,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC;YACzB,CAAC;YACD,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC;QACpD,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IACvC,CAAC;IAED;;;;;OAKG;IACH,aAAa,CAAC,UAAkB;QAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM;YAAE,OAAO;QACpB,MAAM,CAAC,YAAY,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAmB;QACvB,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;YAC7B,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACrB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED;;;;OAIG;IACH,QAAQ,CAAC,UAAkB;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC;QAChE,MAAM,KAAK,GAAsB,mBAAmB,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA,CAAC,CAAC,QAAQ,CAAC;QAC5E,OAAO;YACL,UAAU;YACV,KAAK;YACL,mBAAmB;YACnB,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,UAAU,EAAE,MAAM,CAAC,UAAU;SAC9B,CAAC;IACJ,CAAC;IAEO,UAAU;QAChB,OAAO;YACL,SAAS,EAAE,CAAC;YACZ,YAAY,EAAE,CAAC;YACf,cAAc,EAAE,IAAI;YACpB,aAAa,EAAE,IAAI;YACnB,UAAU,EAAE,CAAC;SACd,CAAC;IACJ,CAAC;CACF;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAA8B,IAAI,yBAAyB,EAAE,CAAC"}
|
|
@@ -22,6 +22,13 @@ export interface LLMJudgeConfig {
|
|
|
22
22
|
temperature?: number;
|
|
23
23
|
/** Custom system prompt for the judge */
|
|
24
24
|
systemPrompt?: string;
|
|
25
|
+
/**
|
|
26
|
+
* Score returned when evaluation fails (provider error, parse failure).
|
|
27
|
+
* Defaults to 0 (fail-closed) so a judge used as a safety gate does not pass
|
|
28
|
+
* on infrastructure failure — matching EmergentJudge. Set to 0.5 to restore
|
|
29
|
+
* the prior neutral-on-error behavior for non-gating quality scoring.
|
|
30
|
+
*/
|
|
31
|
+
errorScore?: number;
|
|
25
32
|
}
|
|
26
33
|
/**
|
|
27
34
|
* Evaluation criteria for LLM judge
|
|
@@ -60,6 +67,7 @@ export declare class LLMJudge {
|
|
|
60
67
|
private readonly providerId?;
|
|
61
68
|
private readonly temperature;
|
|
62
69
|
private readonly systemPrompt;
|
|
70
|
+
private readonly errorScore;
|
|
63
71
|
constructor(config: LLMJudgeConfig);
|
|
64
72
|
/**
|
|
65
73
|
* Judge an AI output against criteria
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LLMJudge.d.ts","sourceRoot":"","sources":["../../../src/safety/evaluation/LLMJudge.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,iDAAiD,CAAC;AAE9F,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEnD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,2BAA2B;IAC3B,WAAW,EAAE,sBAAsB,CAAC;IACpC,+BAA+B;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kBAAkB;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,wDAAwD;IACxD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yCAAyC;IACzC,YAAY,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"LLMJudge.d.ts","sourceRoot":"","sources":["../../../src/safety/evaluation/LLMJudge.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,iDAAiD,CAAC;AAE9F,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEnD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,2BAA2B;IAC3B,WAAW,EAAE,sBAAsB,CAAC;IACpC,+BAA+B;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kBAAkB;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,wDAAwD;IACxD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yCAAyC;IACzC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;OAKG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,qBAAqB;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,sCAAsC;IACtC,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yBAAyB;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,0BAA0B;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,kCAAkC;IAClC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,iCAAiC;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,wBAAwB;IACxB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;CACpB;AAkED;;GAEG;AACH,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyB;IACrD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;gBAExB,MAAM,EAAE,cAAc;IASlC;;OAEG;IACG,KAAK,CACT,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,MAAM,EACpB,cAAc,CAAC,EAAE,MAAM,EACvB,QAAQ,CAAC,EAAE,aAAa,EAAE,GACzB,OAAO,CAAC,cAAc,CAAC;IAmE1B;;OAEG;IACH,YAAY,CAAC,QAAQ,CAAC,EAAE,aAAa,EAAE,GAAG,cAAc;IAQxD;;OAEG;IACG,OAAO,CACX,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,aAAa,EAAE,GACzB,OAAO,CAAC;QACT,MAAM,EAAE,GAAG,GAAG,GAAG,GAAG,KAAK,CAAC;QAC1B,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IAqBF;;OAEG;IACG,UAAU,CACd,WAAW,EAAE,KAAK,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,YAAY,EAAE,MAAM,CAAC;QACrB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC,EACF,QAAQ,CAAC,EAAE,aAAa,EAAE,EAC1B,WAAW,SAAI,GACd,OAAO,CAAC,cAAc,EAAE,CAAC;CAmB7B;AAED;;GAEG;AACH,eAAO,MAAM,gBAAgB;IAC3B,qCAAqC;oBAOhC,aAAa,EAAE;IAEpB,+BAA+B;mBAM1B,aAAa,EAAE;IAEpB,yBAAyB;uBAMpB,aAAa,EAAE;IAEpB,sCAAsC;qBAMjC,aAAa,EAAE;IAEpB,yCAAyC;YAMpC,aAAa,EAAE;CACrB,CAAC"}
|
|
@@ -78,6 +78,7 @@ export class LLMJudge {
|
|
|
78
78
|
this.providerId = config.providerId;
|
|
79
79
|
this.temperature = config.temperature ?? 0.1;
|
|
80
80
|
this.systemPrompt = config.systemPrompt || DEFAULT_JUDGE_PROMPT;
|
|
81
|
+
this.errorScore = config.errorScore ?? 0;
|
|
81
82
|
}
|
|
82
83
|
/**
|
|
83
84
|
* Judge an AI output against criteria
|
|
@@ -126,9 +127,11 @@ Please evaluate the ACTUAL OUTPUT against the criteria and provide your judgment
|
|
|
126
127
|
};
|
|
127
128
|
}
|
|
128
129
|
catch (error) {
|
|
129
|
-
//
|
|
130
|
+
// Fail closed: an errored judge has no signal, so return the configured
|
|
131
|
+
// errorScore (default 0) rather than a misleading neutral 0.5 that would
|
|
132
|
+
// let a `score >= threshold` safety gate pass on infrastructure failure.
|
|
130
133
|
return {
|
|
131
|
-
score:
|
|
134
|
+
score: this.errorScore,
|
|
132
135
|
criteriaScores: {},
|
|
133
136
|
reasoning: `Evaluation error: ${error.message}`,
|
|
134
137
|
feedback: ['Unable to complete evaluation'],
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LLMJudge.js","sourceRoot":"","sources":["../../../src/safety/evaluation/LLMJudge.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;
|
|
1
|
+
{"version":3,"file":"LLMJudge.js","sourceRoot":"","sources":["../../../src/safety/evaluation/LLMJudge.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AA2DH;;GAEG;AACH,MAAM,gBAAgB,GAAoB;IACxC;QACE,IAAI,EAAE,UAAU;QAChB,WAAW,EAAE,qDAAqD;QAClE,MAAM,EAAE,GAAG;QACX,MAAM,EAAE,gEAAgE;KACzE;IACD;QACE,IAAI,EAAE,WAAW;QACjB,WAAW,EAAE,qDAAqD;QAClE,MAAM,EAAE,IAAI;QACZ,MAAM,EAAE,2DAA2D;KACpE;IACD;QACE,IAAI,EAAE,cAAc;QACpB,WAAW,EAAE,4CAA4C;QACzD,MAAM,EAAE,GAAG;QACX,MAAM,EAAE,8DAA8D;KACvE;IACD;QACE,IAAI,EAAE,SAAS;QACf,WAAW,EAAE,gDAAgD;QAC7D,MAAM,EAAE,IAAI;QACZ,MAAM,EAAE,qDAAqD;KAC9D;IACD;QACE,IAAI,EAAE,aAAa;QACnB,WAAW,EAAE,iDAAiD;QAC9D,MAAM,EAAE,GAAG;QACX,MAAM,EAAE,wDAAwD;KACjE;CACF,CAAC;AAEF;;GAEG;AACH,MAAM,oBAAoB,GAAG;;;;;;;;;;;;;;;;;;;;;;;mGAuBsE,CAAC;AAEpG;;GAEG;AACH,MAAM,OAAO,QAAQ;IAQnB,YAAY,MAAsB;QAChC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,aAAa,CAAC;QAC/C,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;QACpC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,GAAG,CAAC;QAC7C,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,oBAAoB,CAAC;QAChE,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK,CACT,KAAa,EACb,YAAoB,EACpB,cAAuB,EACvB,QAA0B;QAE1B,MAAM,YAAY,GAAG,QAAQ,IAAI,gBAAgB,CAAC;QAElD,MAAM,YAAY,GAAG,YAAY;aAC9B,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,WAAW,eAAe,CAAC,CAAC,MAAM,IAAI,oBAAoB,EAAE,CAAC;aACrH,IAAI,CAAC,IAAI,CAAC,CAAC;QAEd,MAAM,WAAW,GAAG;;EAEtB,KAAK;;;EAGL,cAAc,IAAI,6DAA6D;;;EAG/E,YAAY;;;EAGZ,YAAY;;iGAEmF,CAAC;QAE9F,MAAM,QAAQ,GAAkB;YAC9B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,YAAY,EAAE;YAC9C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE;SACvC,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,QAAQ,CAAC;YAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;YAC1D,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,IAAI,KAAK,CAAC,aAAa,UAAU,aAAa,CAAC,CAAC;YACxD,CAAC;YAED,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,kBAAkB,CAClD,IAAI,CAAC,OAAO,EACZ,QAAQ,EACR;gBACE,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,cAAc,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;aACxC,CACF,CAAC;YAEF,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;YAC1D,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAExE,OAAO;gBACL,KAAK,EAAE,MAAM,CAAC,YAAY,IAAI,GAAG;gBACjC,cAAc,EAAE,MAAM,CAAC,cAAc,IAAI,EAAE;gBAC3C,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,uBAAuB;gBACtD,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,EAAE;gBAC/B,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,GAAG;aACrC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,wEAAwE;YACxE,yEAAyE;YACzE,yEAAyE;YACzE,OAAO;gBACL,KAAK,EAAE,IAAI,CAAC,UAAU;gBACtB,cAAc,EAAE,EAAE;gBAClB,SAAS,EAAE,qBAAqB,KAAK,CAAC,OAAO,EAAE;gBAC/C,QAAQ,EAAE,CAAC,+BAA+B,CAAC;gBAC3C,UAAU,EAAE,CAAC;aACd,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,QAA0B;QACrC,OAAO,KAAK,EAAE,MAAc,EAAE,QAAiB,EAAE,WAAsB,EAAE,QAAkC,EAAE,EAAE;YAC7G,MAAM,KAAK,GAAI,QAAQ,EAAE,KAAgB,IAAI,EAAE,CAAC;YAChD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YACnE,OAAO,MAAM,CAAC,KAAK,CAAC;QACtB,CAAC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CACX,KAAa,EACb,OAAe,EACf,OAAe,EACf,QAA0B;QAO1B,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3C,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,CAAC;YAC/C,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,CAAC;SAChD,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,kCAAkC;QAE1D,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;YACjE,MAAM,EAAE,OAAO,CAAC,KAAK;YACrB,MAAM,EAAE,OAAO,CAAC,KAAK;YACrB,SAAS,EAAE,mBAAmB,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,qBAAqB,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KACjG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,SAAS;gBACxB,CAAC,CAAC,qCAAqC;gBACvC,CAAC,CAAC,UAAU,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,gBACpC,EAAE;SACH,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CACd,WAIE,EACF,QAA0B,EAC1B,WAAW,GAAG,CAAC;QAEf,MAAM,OAAO,GAAqB,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC;QAE/B,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;YACxB,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;gBAC3B,IAAI,IAAI,EAAE,CAAC;oBACT,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;oBAC9F,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC;QACpE,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAE3B,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC9B,qCAAqC;IACrC,cAAc,EAAE;QACd,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,+BAA+B,EAAE,MAAM,EAAE,IAAI,EAAE;QACnF,EAAE,IAAI,EAAE,cAAc,EAAE,WAAW,EAAE,kCAAkC,EAAE,MAAM,EAAE,IAAI,EAAE;QACvF,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,wCAAwC,EAAE,MAAM,EAAE,IAAI,EAAE;QACtF,EAAE,IAAI,EAAE,YAAY,EAAE,WAAW,EAAE,mCAAmC,EAAE,MAAM,EAAE,IAAI,EAAE;QACtF,EAAE,IAAI,EAAE,eAAe,EAAE,WAAW,EAAE,iCAAiC,EAAE,MAAM,EAAE,GAAG,EAAE;KACpE;IAEpB,+BAA+B;IAC/B,aAAa,EAAE;QACb,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,0CAA0C,EAAE,MAAM,EAAE,GAAG,EAAE;QAC1F,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,+BAA+B,EAAE,MAAM,EAAE,GAAG,EAAE;QAC/E,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,8BAA8B,EAAE,MAAM,EAAE,GAAG,EAAE;QACjF,EAAE,IAAI,EAAE,WAAW,EAAE,WAAW,EAAE,oCAAoC,EAAE,MAAM,EAAE,GAAG,EAAE;KACnE;IAEpB,yBAAyB;IACzB,iBAAiB,EAAE;QACjB,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,kCAAkC,EAAE,MAAM,EAAE,GAAG,EAAE;QACrF,EAAE,IAAI,EAAE,WAAW,EAAE,WAAW,EAAE,uCAAuC,EAAE,MAAM,EAAE,GAAG,EAAE;QACxF,EAAE,IAAI,EAAE,cAAc,EAAE,WAAW,EAAE,yBAAyB,EAAE,MAAM,EAAE,GAAG,EAAE;QAC7E,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,iCAAiC,EAAE,MAAM,EAAE,GAAG,EAAE;KAC9D;IAEpB,sCAAsC;IACtC,eAAe,EAAE;QACf,EAAE,IAAI,EAAE,YAAY,EAAE,WAAW,EAAE,8BAA8B,EAAE,MAAM,EAAE,GAAG,EAAE;QAChF,EAAE,IAAI,EAAE,WAAW,EAAE,WAAW,EAAE,mCAAmC,EAAE,MAAM,EAAE,IAAI,EAAE;QACrF,EAAE,IAAI,EAAE,YAAY,EAAE,WAAW,EAAE,iCAAiC,EAAE,MAAM,EAAE,IAAI,EAAE;QACpF,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,mCAAmC,EAAE,MAAM,EAAE,GAAG,EAAE;KAC9D;IAEpB,yCAAyC;IACzC,MAAM,EAAE;QACN,EAAE,IAAI,EAAE,cAAc,EAAE,WAAW,EAAE,0CAA0C,EAAE,MAAM,EAAE,GAAG,EAAE;QAC9F,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,+BAA+B,EAAE,MAAM,EAAE,GAAG,EAAE;QAC/E,EAAE,IAAI,EAAE,iBAAiB,EAAE,WAAW,EAAE,0CAA0C,EAAE,MAAM,EAAE,GAAG,EAAE;QACjG,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,gDAAgD,EAAE,MAAM,EAAE,GAAG,EAAE;KACjF;CACrB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@framers/agentos",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.72",
|
|
4
4
|
"description": "AgentOS: open-source TypeScript framework for autonomous AI agents. Unified graph orchestration, cognitive memory, runtime tool forging, multi-tier guardrails, voice pipeline, and 11 LLM providers.",
|
|
5
5
|
"homepage": "https://agentos.sh",
|
|
6
6
|
"repository": {
|