@tangle-network/agent-eval 0.205.2 → 0.207.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/dist/adapters/http.d.ts +1 -1
- package/dist/analyst/index.d.ts +1 -1
- package/dist/analyst/index.js +1 -1
- package/dist/{benchmark-command-DuDTN5ID.js → benchmark-command-DtP94cOB.js} +2 -2
- package/dist/{benchmark-command-DuDTN5ID.js.map → benchmark-command-DtP94cOB.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +2 -2
- package/dist/campaign/index.d.ts +3 -3
- package/dist/campaign/index.js +2 -2
- package/dist/cli.js +3 -2
- package/dist/cli.js.map +1 -1
- package/dist/contract/index.d.ts +4 -4
- package/dist/{define-agent-eval-CKB6wyML.d.ts → define-agent-eval-B2YV-RDE.d.ts} +3 -3
- package/dist/{define-agent-eval-CKB6wyML.d.ts.map → define-agent-eval-B2YV-RDE.d.ts.map} +1 -1
- package/dist/{index-BRdZ0oJo.d.ts → index-CojsA_EE.d.ts} +2 -2
- package/dist/{index-BRdZ0oJo.d.ts.map → index-CojsA_EE.d.ts.map} +1 -1
- package/dist/{index-R8Xhod8e.d.ts → index-DVDiJIhi.d.ts} +3 -3
- package/dist/{index-R8Xhod8e.d.ts.map → index-DVDiJIhi.d.ts.map} +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/meta-eval/index.d.ts +69 -0
- package/dist/meta-eval/index.d.ts.map +1 -1
- package/dist/meta-eval/index.js +135 -1
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/rl.d.ts +12 -12
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +17 -17
- package/dist/rl.js.map +1 -1
- package/dist/search/index.d.ts +8 -2
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +1 -1
- package/dist/{search-kernel-DNpsHb8P.d.ts → search-kernel-CZ8KJQRw.d.ts} +6 -3
- package/dist/{search-kernel-DNpsHb8P.d.ts.map → search-kernel-CZ8KJQRw.d.ts.map} +1 -1
- package/dist/systemone-protocol-DKoi1NdK.d.ts +132 -0
- package/dist/systemone-protocol-DKoi1NdK.d.ts.map +1 -0
- package/dist/systemone-protocol.d.ts +2 -0
- package/dist/systemone-protocol.js +214 -0
- package/dist/systemone-protocol.js.map +1 -0
- package/dist/{jev-review-Cw06AKuF.d.ts → systemone-review-DGTKkLMG.d.ts} +16 -16
- package/dist/systemone-review-DGTKkLMG.d.ts.map +1 -0
- package/dist/{jev-review-DZEjDzlF.js → systemone-review-DTClwgmE.js} +11 -11
- package/dist/systemone-review-DTClwgmE.js.map +1 -0
- package/dist/systemone.d.ts +60 -0
- package/dist/systemone.d.ts.map +1 -0
- package/dist/{jev.js → systemone.js} +16 -16
- package/dist/systemone.js.map +1 -0
- package/dist/{tree-By7LtLUu.js → tree-CN8mlzUO.js} +11 -3
- package/dist/{tree-By7LtLUu.js.map → tree-CN8mlzUO.js.map} +1 -1
- package/docs/changes/jev-evidence-review.md +1 -1
- package/docs/changes/systemone-provider-neutral.md +36 -0
- package/docs/public-api.md +96 -70
- package/docs/search-ledger.md +9 -3
- package/docs/{jev-behavior-review.md → systemone-behavior-review.md} +17 -17
- package/docs/{jev-integrations.md → systemone-integrations.md} +4 -4
- package/docs/{jev-replay.md → systemone-replay.md} +6 -6
- package/docs/{jev.md → systemone.md} +96 -33
- package/package.json +9 -9
- package/dist/jev-protocol-BQLxwbqD.d.ts +0 -99
- package/dist/jev-protocol-BQLxwbqD.d.ts.map +0 -1
- package/dist/jev-protocol.d.ts +0 -2
- package/dist/jev-protocol.js +0 -114
- package/dist/jev-protocol.js.map +0 -1
- package/dist/jev-review-Cw06AKuF.d.ts.map +0 -1
- package/dist/jev-review-DZEjDzlF.js.map +0 -1
- package/dist/jev.d.ts +0 -60
- package/dist/jev.d.ts.map +0 -1
- package/dist/jev.js.map +0 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { i as AnalystFinding } from "./types-6VEehmEg.js";
|
|
2
|
-
import { f as
|
|
2
|
+
import { f as SystemOneResult, o as SystemOneQuestions, u as SystemOneRequest } from "./systemone-protocol-DKoi1NdK.js";
|
|
3
3
|
import { z } from "zod";
|
|
4
|
-
//#region src/
|
|
4
|
+
//#region src/systemone-review.d.ts
|
|
5
5
|
declare const checkSchema: z.ZodObject<{
|
|
6
6
|
claim: z.ZodString;
|
|
7
7
|
area: z.ZodString;
|
|
@@ -35,17 +35,17 @@ declare const checkSchema: z.ZodObject<{
|
|
|
35
35
|
}, z.core.$strict>>;
|
|
36
36
|
}, z.core.$strict>;
|
|
37
37
|
/** Optional interpretation of native choices/scores. Policy and evidence belong to the caller. */
|
|
38
|
-
type
|
|
39
|
-
interface
|
|
38
|
+
type SystemOneReviewCheck = z.infer<typeof checkSchema>;
|
|
39
|
+
interface SystemOneReviewInput<Q extends SystemOneQuestions = SystemOneQuestions> {
|
|
40
40
|
version: string;
|
|
41
|
-
request:
|
|
42
|
-
checks: { [K in keyof Q]:
|
|
41
|
+
request: SystemOneRequest<Q>;
|
|
42
|
+
checks: { [K in keyof Q]: SystemOneReviewCheck; };
|
|
43
43
|
}
|
|
44
|
-
interface
|
|
44
|
+
interface PreparedSystemOneReview<Q extends SystemOneQuestions = SystemOneQuestions> extends SystemOneReviewInput<Q> {
|
|
45
45
|
/** Binds questions, option order, context, evidence, thresholds and definition version. */
|
|
46
46
|
digest: string;
|
|
47
47
|
}
|
|
48
|
-
interface
|
|
48
|
+
interface SystemOneReviewAssessment {
|
|
49
49
|
question: string;
|
|
50
50
|
status: 'supported' | 'refuted' | 'unresolved';
|
|
51
51
|
reason: 'support-threshold' | 'refutation-threshold' | 'missing-evidence' | 'partial-negative-evidence' | 'below-threshold' | 'ambiguous-thresholds';
|
|
@@ -54,23 +54,23 @@ interface JevReviewAssessment {
|
|
|
54
54
|
refutationProbability: number;
|
|
55
55
|
unresolvedProbability: number;
|
|
56
56
|
modelConfidence: number | null;
|
|
57
|
-
check:
|
|
57
|
+
check: SystemOneReviewCheck;
|
|
58
58
|
}
|
|
59
|
-
interface
|
|
59
|
+
interface SystemOneReviewReport {
|
|
60
60
|
reviewDigest: string;
|
|
61
61
|
requestedModel: string;
|
|
62
62
|
servedModel: string;
|
|
63
|
-
assessments:
|
|
63
|
+
assessments: SystemOneReviewAssessment[];
|
|
64
64
|
}
|
|
65
65
|
/** Validate before paid admission; make a detached, frozen snapshot without rewriting native JSON. */
|
|
66
|
-
declare function
|
|
66
|
+
declare function prepareSystemOneReview<const Q extends SystemOneQuestions>(input: SystemOneReviewInput<Q>): PreparedSystemOneReview<Q>;
|
|
67
67
|
/** Re-score the same retained distribution without new inference or any authorization effect. */
|
|
68
|
-
declare function
|
|
68
|
+
declare function assessSystemOneReview<Q extends SystemOneQuestions>(review: PreparedSystemOneReview<Q>, raw: unknown): SystemOneReviewReport;
|
|
69
69
|
/** Ordinary analyst findings; a negative review is not a global safety certificate. */
|
|
70
|
-
declare function
|
|
70
|
+
declare function systemOneReviewFindings<Q extends SystemOneQuestions>(review: PreparedSystemOneReview<Q>, result: SystemOneResult<Q>, options: {
|
|
71
71
|
analystId: string;
|
|
72
72
|
producedAt?: string;
|
|
73
73
|
}): AnalystFinding[];
|
|
74
74
|
//#endregion
|
|
75
|
-
export {
|
|
76
|
-
//# sourceMappingURL=
|
|
75
|
+
export { SystemOneReviewReport as a, systemOneReviewFindings as c, SystemOneReviewInput as i, SystemOneReviewAssessment as n, assessSystemOneReview as o, SystemOneReviewCheck as r, prepareSystemOneReview as s, PreparedSystemOneReview as t };
|
|
76
|
+
//# sourceMappingURL=systemone-review-DGTKkLMG.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"systemone-review-DGTKkLMG.d.ts","names":[],"sources":["../src/systemone-review.ts"],"mappings":";;;;cAcM,aAAW,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0Bb,EAAA,KAAA;;KAGQ,uBAAuB,EAAE,aAAa;UACjC,qBAAqB,UAAU,qBAAqB;EACnE;EACA,SAAS,iBAAiB;EAC1B,WAAW,WAAW,IAAI;;UAGX,wBAAwB,UAAU,qBAAqB,4BAC9D,qBAAqB;;EAE7B;;UAGe;EACf;EACA;EACA;;EAQA;EACA;EACA;EACA;EACA,OAAO;;UAGQ;EACf;EACA;EACA;EACA,aAAa;;;iBAWC,6BAA6B,UAAU,oBACrD,OAAO,qBAAqB,KAC3B,wBAAwB;;iBAuCX,sBAAsB,UAAU,oBAC9C,QAAQ,wBAAwB,IAChC,eACC;;iBAyDa,wBAAwB,UAAU,oBAChD,QAAQ,wBAAwB,IAChC,QAAQ,gBAAgB,IACxB;EAAW;EAAmB;IAC7B"}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { a as hashCanonical, o as jsonDocument, r as canonicalString } from "./canonical-BsZVgBhW.js";
|
|
2
2
|
import { t as deepFreezeCanonicalJson } from "./deep-freeze-CurcOQg3.js";
|
|
3
3
|
import { r as makeFinding, t as ANALYST_SEVERITIES } from "./types-CW5b90JH.js";
|
|
4
|
-
import {
|
|
4
|
+
import { parseSystemOneRequest, parseSystemOneResult } from "./systemone-protocol.js";
|
|
5
5
|
import { z } from "zod";
|
|
6
|
-
//#region src/
|
|
6
|
+
//#region src/systemone-review.ts
|
|
7
7
|
const name = z.string().trim().min(1);
|
|
8
8
|
const probability = z.number().finite().gt(0).max(1);
|
|
9
9
|
const checkSchema = z.object({
|
|
@@ -39,9 +39,9 @@ function alternatives(question) {
|
|
|
39
39
|
return question.type === "choice" ? Object.keys(question.criteria) : question.criteria.map((_, index) => String(index));
|
|
40
40
|
}
|
|
41
41
|
/** Validate before paid admission; make a detached, frozen snapshot without rewriting native JSON. */
|
|
42
|
-
function
|
|
42
|
+
function prepareSystemOneReview(input) {
|
|
43
43
|
if (!input.version?.trim()) throw new TypeError("Review version is required");
|
|
44
|
-
|
|
44
|
+
parseSystemOneRequest(input.request);
|
|
45
45
|
const document = jsonDocument(input);
|
|
46
46
|
canonicalString(document);
|
|
47
47
|
const snapshot = structuredClone(document);
|
|
@@ -65,11 +65,11 @@ function prepareJevReview(input) {
|
|
|
65
65
|
});
|
|
66
66
|
}
|
|
67
67
|
/** Re-score the same retained distribution without new inference or any authorization effect. */
|
|
68
|
-
function
|
|
68
|
+
function assessSystemOneReview(review, raw) {
|
|
69
69
|
const { digest, ...input } = review;
|
|
70
|
-
const prepared =
|
|
70
|
+
const prepared = prepareSystemOneReview(input);
|
|
71
71
|
if (prepared.digest !== digest) throw new TypeError("Review definition changed after preparation");
|
|
72
|
-
const result =
|
|
72
|
+
const result = parseSystemOneResult(raw, prepared.request);
|
|
73
73
|
const assessments = Object.entries(prepared.checks).map(([question, check]) => {
|
|
74
74
|
const answer = result.answers[question];
|
|
75
75
|
const probabilities = answer.type === "noul" ? {
|
|
@@ -114,9 +114,9 @@ function assessJevReview(review, raw) {
|
|
|
114
114
|
};
|
|
115
115
|
}
|
|
116
116
|
/** Ordinary analyst findings; a negative review is not a global safety certificate. */
|
|
117
|
-
function
|
|
117
|
+
function systemOneReviewFindings(review, result, options) {
|
|
118
118
|
if (!options.analystId.trim()) throw new TypeError("Analyst id is required");
|
|
119
|
-
const report =
|
|
119
|
+
const report = assessSystemOneReview(review, result);
|
|
120
120
|
return report.assessments.filter((item) => item.status !== "refuted").map((item) => makeFinding({
|
|
121
121
|
analyst_id: options.analystId,
|
|
122
122
|
...options.producedAt === void 0 ? {} : { produced_at: options.producedAt },
|
|
@@ -152,6 +152,6 @@ function jevReviewFindings(review, result, options) {
|
|
|
152
152
|
}));
|
|
153
153
|
}
|
|
154
154
|
//#endregion
|
|
155
|
-
export {
|
|
155
|
+
export { prepareSystemOneReview as n, systemOneReviewFindings as r, assessSystemOneReview as t };
|
|
156
156
|
|
|
157
|
-
//# sourceMappingURL=
|
|
157
|
+
//# sourceMappingURL=systemone-review-DTClwgmE.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"systemone-review-DTClwgmE.js","names":[],"sources":["../src/systemone-review.ts"],"sourcesContent":["import { z } from 'zod'\nimport {\n ANALYST_SEVERITIES,\n type AnalystFinding,\n type EvidenceRef,\n makeFinding,\n} from './analyst/types'\nimport { canonicalString, hashCanonical, jsonDocument } from './ledger-core/canonical'\nimport { deepFreezeCanonicalJson } from './ledger-core/deep-freeze'\nimport type { SystemOneQuestions, SystemOneRequest, SystemOneResult } from './systemone-protocol'\nimport { parseSystemOneRequest, parseSystemOneResult } from './systemone-protocol'\n\nconst name = z.string().trim().min(1)\nconst probability = z.number().finite().gt(0).max(1)\nconst checkSchema = z\n .object({\n claim: name,\n area: name,\n subject: name,\n severity: z.enum(ANALYST_SEVERITIES),\n /** Choice labels or score-level keys, never scores invented from category order. */\n supports: z.array(z.string()).min(1),\n refutes: z.array(z.string()).min(1),\n supportAtLeast: probability,\n refuteAtLeast: probability,\n /** Host-supplied coverage of the question's required evidence, not model self-assessment. */\n coverage: z.enum(['complete', 'partial', 'missing']),\n evidence: z.array(\n z\n .object({\n kind: z.enum(['span', 'event', 'artifact', 'finding', 'metric']),\n uri: name,\n excerpt: z.string().optional(),\n })\n .strict(),\n ),\n })\n .strict()\n .refine((value) => value.supportAtLeast + value.refuteAtLeast > 1, {\n message: 'Support and refutation thresholds must not overlap',\n })\n\n/** Optional interpretation of native choices/scores. Policy and evidence belong to the caller. */\nexport type SystemOneReviewCheck = z.infer<typeof checkSchema>\nexport interface SystemOneReviewInput<Q extends SystemOneQuestions = SystemOneQuestions> {\n version: string\n request: SystemOneRequest<Q>\n checks: { [K in keyof Q]: SystemOneReviewCheck }\n}\n\nexport interface PreparedSystemOneReview<Q extends SystemOneQuestions = SystemOneQuestions>\n extends SystemOneReviewInput<Q> {\n /** Binds questions, option order, context, evidence, thresholds and definition version. */\n digest: string\n}\n\nexport interface SystemOneReviewAssessment {\n question: string\n status: 'supported' | 'refuted' | 'unresolved'\n reason:\n | 'support-threshold'\n | 'refutation-threshold'\n | 'missing-evidence'\n | 'partial-negative-evidence'\n | 'below-threshold'\n | 'ambiguous-thresholds'\n /** Unnormalized marginal mass of caller-designated native alternatives. Not calibrated risk. */\n supportProbability: number\n refutationProbability: number\n unresolvedProbability: number\n modelConfidence: number | null\n check: SystemOneReviewCheck\n}\n\nexport interface SystemOneReviewReport {\n reviewDigest: string\n requestedModel: string\n servedModel: string\n assessments: SystemOneReviewAssessment[]\n}\n\nfunction alternatives(question: SystemOneQuestions[string]): string[] {\n if (question.type === 'noul') return ['true', 'false']\n return question.type === 'choice'\n ? Object.keys(question.criteria)\n : question.criteria.map((_, index) => String(index))\n}\n\n/** Validate before paid admission; make a detached, frozen snapshot without rewriting native JSON. */\nexport function prepareSystemOneReview<const Q extends SystemOneQuestions>(\n input: SystemOneReviewInput<Q>,\n): PreparedSystemOneReview<Q> {\n if (!input.version?.trim()) throw new TypeError('Review version is required')\n parseSystemOneRequest(input.request)\n // Document form drops only optional undefined object fields; canonicalization rejects loss.\n const document = jsonDocument(input)\n canonicalString(document)\n // Canonical bytes are for hashing, not dispatch: option order can affect a classifier.\n const snapshot = structuredClone(document) as SystemOneReviewInput<Q>\n const keys = Object.keys(snapshot.request.questions)\n if (\n !snapshot.checks ||\n Object.keys(snapshot.checks).length !== keys.length ||\n keys.some((key) => !Object.hasOwn(snapshot.checks, key))\n ) {\n throw new TypeError('Review checks must match the exact question names')\n }\n for (const key of keys) {\n const check = checkSchema.parse(snapshot.checks[key])\n const labels = [...check.supports, ...check.refutes]\n const allowed = alternatives(snapshot.request.questions[key]!)\n if (\n new Set(labels).size !== labels.length ||\n labels.some((label) => !allowed.includes(label))\n ) {\n throw new TypeError('Review alternatives must exist and be disjoint')\n }\n if (check.coverage !== 'missing' && check.evidence.length === 0) {\n throw new TypeError('Review evidence references are required for non-missing coverage')\n }\n // Validation must not normalize a label into a different native option.\n if (canonicalString(jsonDocument(check)) !== canonicalString(snapshot.checks[key])) {\n throw new TypeError('Review metadata must not require normalization')\n }\n }\n const order = keys.map((key) => [key, alternatives(snapshot.request.questions[key]!)])\n return deepFreezeCanonicalJson({ ...snapshot, digest: hashCanonical({ snapshot, order }) })\n}\n\n/** Re-score the same retained distribution without new inference or any authorization effect. */\nexport function assessSystemOneReview<Q extends SystemOneQuestions>(\n review: PreparedSystemOneReview<Q>,\n raw: unknown,\n): SystemOneReviewReport {\n const { digest, ...input } = review\n const prepared = prepareSystemOneReview(input)\n if (prepared.digest !== digest) throw new TypeError('Review definition changed after preparation')\n const result = parseSystemOneResult(raw, prepared.request)\n const assessments = Object.entries(prepared.checks).map(([question, check]) => {\n const answer = result.answers[question]!\n const probabilities: Record<string, number> =\n answer.type === 'noul' ? { true: answer.noul, false: 1 - answer.noul } : answer.probabilities\n const sum = (labels: string[]) =>\n labels.reduce((total, label) => total + probabilities[label]!, 0)\n const supportProbability = sum(check.supports)\n const refutationProbability = sum(check.refutes)\n const used = new Set([...check.supports, ...check.refutes])\n const unresolvedProbability = sum(\n Object.keys(probabilities).filter((label) => !used.has(label)),\n )\n let status: SystemOneReviewAssessment['status'] = 'unresolved'\n let reason: SystemOneReviewAssessment['reason'] = 'below-threshold'\n if (supportProbability > 1 || refutationProbability > 1 || unresolvedProbability > 1) {\n throw new TypeError('Review probability mass exceeds one; do not silently normalize it')\n }\n if (check.coverage === 'missing') reason = 'missing-evidence'\n else if (\n supportProbability >= check.supportAtLeast &&\n refutationProbability >= check.refuteAtLeast\n ) {\n reason = 'ambiguous-thresholds'\n } else if (supportProbability >= check.supportAtLeast) {\n status = 'supported'\n reason = 'support-threshold'\n } else if (refutationProbability >= check.refuteAtLeast) {\n if (check.coverage === 'complete') {\n status = 'refuted'\n reason = 'refutation-threshold'\n } else reason = 'partial-negative-evidence'\n }\n return {\n question,\n status,\n reason,\n supportProbability,\n refutationProbability,\n unresolvedProbability,\n modelConfidence: answer.type === 'noul' ? null : answer.confidence,\n check: structuredClone(check),\n }\n })\n return {\n reviewDigest: digest,\n requestedModel: review.request.model,\n servedModel: result.model,\n assessments,\n }\n}\n\n/** Ordinary analyst findings; a negative review is not a global safety certificate. */\nexport function systemOneReviewFindings<Q extends SystemOneQuestions>(\n review: PreparedSystemOneReview<Q>,\n result: SystemOneResult<Q>,\n options: { analystId: string; producedAt?: string },\n): AnalystFinding[] {\n if (!options.analystId.trim()) throw new TypeError('Analyst id is required')\n const report = assessSystemOneReview(review, result)\n return report.assessments\n .filter((item) => item.status !== 'refuted')\n .map((item) =>\n makeFinding({\n analyst_id: options.analystId,\n ...(options.producedAt === undefined ? {} : { produced_at: options.producedAt }),\n subject: item.check.subject,\n area: item.status === 'unresolved' ? 'assessment-coverage' : item.check.area,\n severity: item.status === 'unresolved' ? 'info' : item.check.severity,\n claim:\n item.status === 'supported'\n ? `Model-supported hypothesis: ${item.check.claim}`\n : `Unresolved review: ${item.check.claim}`,\n evidence_refs: structuredClone(item.check.evidence) as EvidenceRef[],\n // For unresolved findings, confidence describes the deterministic unresolved status only.\n confidence: item.status === 'supported' ? item.supportProbability : 1,\n derived_from_judge: true,\n id_basis: hashCanonical([item.question, item.status, review.version, item.check.claim]),\n metadata: {\n review_digest: report.reviewDigest,\n question: item.question,\n requested_model: report.requestedModel,\n served_model: report.servedModel,\n assessment: item.status,\n assessment_reason: item.reason,\n evidence_coverage: item.check.coverage,\n support_probability: item.supportProbability,\n refutation_probability: item.refutationProbability,\n unresolved_probability: item.unresolvedProbability,\n model_confidence: item.modelConfidence,\n calibration: 'not-established',\n confidence_basis:\n item.status === 'supported' ? 'model-support-mass' : 'deterministic-status',\n native_answer: structuredClone(result.answers[item.question]),\n },\n }),\n )\n}\n"],"mappings":";;;;;;AAYA,MAAM,OAAO,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC;AACpC,MAAM,cAAc,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;AACnD,MAAM,cAAc,EACjB,OAAO;CACN,OAAO;CACP,MAAM;CACN,SAAS;CACT,UAAU,EAAE,KAAK,kBAAkB;;CAEnC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;CACnC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;CAClC,gBAAgB;CAChB,eAAe;;CAEf,UAAU,EAAE,KAAK;EAAC;EAAY;EAAW;CAAS,CAAC;CACnD,UAAU,EAAE,MACV,EACG,OAAO;EACN,MAAM,EAAE,KAAK;GAAC;GAAQ;GAAS;GAAY;GAAW;EAAQ,CAAC;EAC/D,KAAK;EACL,SAAS,EAAE,OAAO,CAAC,CAAC,SAAS;CAC/B,CAAC,CAAC,CACD,OAAO,CACZ;AACF,CAAC,CAAC,CACD,OAAO,CAAC,CACR,QAAQ,UAAU,MAAM,iBAAiB,MAAM,gBAAgB,GAAG,EACjE,SAAS,qDACX,CAAC;AAyCH,SAAS,aAAa,UAAgD;CACpE,IAAI,SAAS,SAAS,QAAQ,OAAO,CAAC,QAAQ,OAAO;CACrD,OAAO,SAAS,SAAS,WACrB,OAAO,KAAK,SAAS,QAAQ,IAC7B,SAAS,SAAS,KAAK,GAAG,UAAU,OAAO,KAAK,CAAC;AACvD;;AAGA,SAAgB,uBACd,OAC4B;CAC5B,IAAI,CAAC,MAAM,SAAS,KAAK,GAAG,MAAM,IAAI,UAAU,4BAA4B;CAC5E,sBAAsB,MAAM,OAAO;CAEnC,MAAM,WAAW,aAAa,KAAK;CACnC,gBAAgB,QAAQ;CAExB,MAAM,WAAW,gBAAgB,QAAQ;CACzC,MAAM,OAAO,OAAO,KAAK,SAAS,QAAQ,SAAS;CACnD,IACE,CAAC,SAAS,UACV,OAAO,KAAK,SAAS,MAAM,CAAC,CAAC,WAAW,KAAK,UAC7C,KAAK,MAAM,QAAQ,CAAC,OAAO,OAAO,SAAS,QAAQ,GAAG,CAAC,GAEvD,MAAM,IAAI,UAAU,mDAAmD;CAEzE,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,QAAQ,YAAY,MAAM,SAAS,OAAO,IAAI;EACpD,MAAM,SAAS,CAAC,GAAG,MAAM,UAAU,GAAG,MAAM,OAAO;EACnD,MAAM,UAAU,aAAa,SAAS,QAAQ,UAAU,IAAK;EAC7D,IACE,IAAI,IAAI,MAAM,CAAC,CAAC,SAAS,OAAO,UAChC,OAAO,MAAM,UAAU,CAAC,QAAQ,SAAS,KAAK,CAAC,GAE/C,MAAM,IAAI,UAAU,gDAAgD;EAEtE,IAAI,MAAM,aAAa,aAAa,MAAM,SAAS,WAAW,GAC5D,MAAM,IAAI,UAAU,kEAAkE;EAGxF,IAAI,gBAAgB,aAAa,KAAK,CAAC,MAAM,gBAAgB,SAAS,OAAO,IAAI,GAC/E,MAAM,IAAI,UAAU,gDAAgD;CAExE;CACA,MAAM,QAAQ,KAAK,KAAK,QAAQ,CAAC,KAAK,aAAa,SAAS,QAAQ,UAAU,IAAK,CAAC,CAAC;CACrF,OAAO,wBAAwB;EAAE,GAAG;EAAU,QAAQ,cAAc;GAAE;GAAU;EAAM,CAAC;CAAE,CAAC;AAC5F;;AAGA,SAAgB,sBACd,QACA,KACuB;CACvB,MAAM,EAAE,QAAQ,GAAG,UAAU;CAC7B,MAAM,WAAW,uBAAuB,KAAK;CAC7C,IAAI,SAAS,WAAW,QAAQ,MAAM,IAAI,UAAU,6CAA6C;CACjG,MAAM,SAAS,qBAAqB,KAAK,SAAS,OAAO;CACzD,MAAM,cAAc,OAAO,QAAQ,SAAS,MAAM,CAAC,CAAC,KAAK,CAAC,UAAU,WAAW;EAC7E,MAAM,SAAS,OAAO,QAAQ;EAC9B,MAAM,gBACJ,OAAO,SAAS,SAAS;GAAE,MAAM,OAAO;GAAM,OAAO,IAAI,OAAO;EAAK,IAAI,OAAO;EAClF,MAAM,OAAO,WACX,OAAO,QAAQ,OAAO,UAAU,QAAQ,cAAc,QAAS,CAAC;EAClE,MAAM,qBAAqB,IAAI,MAAM,QAAQ;EAC7C,MAAM,wBAAwB,IAAI,MAAM,OAAO;EAC/C,MAAM,uBAAO,IAAI,IAAI,CAAC,GAAG,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC;EAC1D,MAAM,wBAAwB,IAC5B,OAAO,KAAK,aAAa,CAAC,CAAC,QAAQ,UAAU,CAAC,KAAK,IAAI,KAAK,CAAC,CAC/D;EACA,IAAI,SAA8C;EAClD,IAAI,SAA8C;EAClD,IAAI,qBAAqB,KAAK,wBAAwB,KAAK,wBAAwB,GACjF,MAAM,IAAI,UAAU,mEAAmE;EAEzF,IAAI,MAAM,aAAa,WAAW,SAAS;OACtC,IACH,sBAAsB,MAAM,kBAC5B,yBAAyB,MAAM,eAE/B,SAAS;OACJ,IAAI,sBAAsB,MAAM,gBAAgB;GACrD,SAAS;GACT,SAAS;EACX,OAAO,IAAI,yBAAyB,MAAM,eAAe;GACvD,IAAI,MAAM,aAAa,YAAY;IACjC,SAAS;IACT,SAAS;GACX,OAAO,SAAS;EAClB;EACA,OAAO;GACL;GACA;GACA;GACA;GACA;GACA;GACA,iBAAiB,OAAO,SAAS,SAAS,OAAO,OAAO;GACxD,OAAO,gBAAgB,KAAK;EAC9B;CACF,CAAC;CACD,OAAO;EACL,cAAc;EACd,gBAAgB,OAAO,QAAQ;EAC/B,aAAa,OAAO;EACpB;CACF;AACF;;AAGA,SAAgB,wBACd,QACA,QACA,SACkB;CAClB,IAAI,CAAC,QAAQ,UAAU,KAAK,GAAG,MAAM,IAAI,UAAU,wBAAwB;CAC3E,MAAM,SAAS,sBAAsB,QAAQ,MAAM;CACnD,OAAO,OAAO,YACX,QAAQ,SAAS,KAAK,WAAW,SAAS,CAAC,CAC3C,KAAK,SACJ,YAAY;EACV,YAAY,QAAQ;EACpB,GAAI,QAAQ,eAAe,KAAA,IAAY,CAAC,IAAI,EAAE,aAAa,QAAQ,WAAW;EAC9E,SAAS,KAAK,MAAM;EACpB,MAAM,KAAK,WAAW,eAAe,wBAAwB,KAAK,MAAM;EACxE,UAAU,KAAK,WAAW,eAAe,SAAS,KAAK,MAAM;EAC7D,OACE,KAAK,WAAW,cACZ,+BAA+B,KAAK,MAAM,UAC1C,sBAAsB,KAAK,MAAM;EACvC,eAAe,gBAAgB,KAAK,MAAM,QAAQ;EAElD,YAAY,KAAK,WAAW,cAAc,KAAK,qBAAqB;EACpE,oBAAoB;EACpB,UAAU,cAAc;GAAC,KAAK;GAAU,KAAK;GAAQ,OAAO;GAAS,KAAK,MAAM;EAAK,CAAC;EACtF,UAAU;GACR,eAAe,OAAO;GACtB,UAAU,KAAK;GACf,iBAAiB,OAAO;GACxB,cAAc,OAAO;GACrB,YAAY,KAAK;GACjB,mBAAmB,KAAK;GACxB,mBAAmB,KAAK,MAAM;GAC9B,qBAAqB,KAAK;GAC1B,wBAAwB,KAAK;GAC7B,wBAAwB,KAAK;GAC7B,kBAAkB,KAAK;GACvB,aAAa;GACb,kBACE,KAAK,WAAW,cAAc,uBAAuB;GACvD,eAAe,gBAAgB,OAAO,QAAQ,KAAK,SAAS;EAC9D;CACF,CAAC,CACH;AACJ"}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { b as CustomTokenPricing, g as CostReceiptInput } from "./cost-ledger-DbQdN3nO.js";
|
|
2
|
+
import { i as AnalystFinding, n as AnalystContext, t as Analyst } from "./types-6VEehmEg.js";
|
|
3
|
+
import { R as Scenario, S as JudgeConfig, w as JudgeScore } from "./types-ZUy5HGY5.js";
|
|
4
|
+
import { EvaluationAnalystOptions, EvaluationContext, EvaluationJudgeOptions, EvaluationResult, Evaluator, EvaluatorOptions, asAnalyst, asJudge, createEvaluator } from "./evaluation.js";
|
|
5
|
+
import { S as systemOneUsage, _ as parseSystemOneQuestions, a as SystemOneQuestion, b as parseSystemOneResult, c as SystemOneRecordedQuestions, d as SystemOneResponseError, f as SystemOneResult, g as normalizeSystemOneResult, h as SystemOneState, i as SystemOneJson, l as SystemOneRecordedRequest, m as SystemOneScoreCriteria, n as SystemOneAnswerFor, o as SystemOneQuestions, p as SystemOneResultProfile, r as SystemOneDescription, s as SystemOneRecordedQuestion, t as SystemOneAnswer, u as SystemOneRequest, v as parseSystemOneRecordedRequest, x as systemOneConfidence, y as parseSystemOneRequest } from "./systemone-protocol-DKoi1NdK.js";
|
|
6
|
+
import { a as SystemOneReviewReport, c as systemOneReviewFindings, i as SystemOneReviewInput, n as SystemOneReviewAssessment, o as assessSystemOneReview, r as SystemOneReviewCheck, s as prepareSystemOneReview, t as PreparedSystemOneReview } from "./systemone-review-DGTKkLMG.js";
|
|
7
|
+
//#region src/systemone.d.ts
|
|
8
|
+
export type SystemOneEvaluate = EvaluatorOptions<SystemOneRequest, unknown>['execute'];
|
|
9
|
+
export interface SystemOneEvaluatorOptions {
|
|
10
|
+
evaluate: SystemOneEvaluate;
|
|
11
|
+
costLedger?: EvaluatorOptions<SystemOneRequest, unknown>['costLedger'];
|
|
12
|
+
maximumCharge?: EvaluatorOptions<SystemOneRequest, unknown>['maximumCharge'];
|
|
13
|
+
pricing?: CustomTokenPricing;
|
|
14
|
+
receipt?: (response: unknown) => CostReceiptInput;
|
|
15
|
+
receiptFromError?: (error: Error) => CostReceiptInput | undefined;
|
|
16
|
+
/** Optional deployment policy; the protocol adapter does not hardcode model aliases. */
|
|
17
|
+
acceptModel?: (requested: string, served: string) => boolean;
|
|
18
|
+
}
|
|
19
|
+
/** Configure transport once; supply native state, questions, and model on every invocation. */
|
|
20
|
+
export declare function systemOneEvaluator(options: SystemOneEvaluatorOptions): <const Q extends SystemOneQuestions>(request: SystemOneRequest<Q>, context?: EvaluationContext) => Promise<EvaluationResult<SystemOneResult<Q>>>;
|
|
21
|
+
type Input<A, S extends Scenario> = {
|
|
22
|
+
artifact: A;
|
|
23
|
+
scenario: S;
|
|
24
|
+
};
|
|
25
|
+
type QuestionSource<I, Q extends SystemOneQuestions> = Q | ((input: I) => Q | Promise<Q>);
|
|
26
|
+
export interface SystemOneOptions extends SystemOneEvaluatorOptions {
|
|
27
|
+
model: string;
|
|
28
|
+
version: string;
|
|
29
|
+
questions: SystemOneQuestions;
|
|
30
|
+
}
|
|
31
|
+
export interface SystemOneJudgeOptions<A, S extends Scenario = Scenario, Q extends SystemOneQuestions = SystemOneQuestions> extends SystemOneEvaluatorOptions {
|
|
32
|
+
model: string;
|
|
33
|
+
version: string;
|
|
34
|
+
questions: QuestionSource<Input<A, S>, Q>;
|
|
35
|
+
renderState: (input: Input<A, S>, context: EvaluationContext) => SystemOneState | Promise<SystemOneState>;
|
|
36
|
+
dimensions?: JudgeConfig<A, S>['dimensions'];
|
|
37
|
+
weights?: Record<string, number>;
|
|
38
|
+
map?: (value: SystemOneResult<Q>, input: Input<A, S>) => JudgeScore | Promise<JudgeScore>;
|
|
39
|
+
record?: (result: EvaluationResult<SystemOneResult<Q>>, input: Input<A, S>) => void | Promise<void>;
|
|
40
|
+
appliesTo?: (scenario: S) => boolean;
|
|
41
|
+
}
|
|
42
|
+
/** Optional convention. Choice utilities and non-uniform scales belong in an explicit map. */
|
|
43
|
+
export declare function normalizedSystemOneScore(result: SystemOneResult, weights?: Record<string, number>): JudgeScore;
|
|
44
|
+
/** Convenience around the public evaluator. No domain questions, thresholds, or prose. */
|
|
45
|
+
export declare function systemOneJudge<A, S extends Scenario = Scenario, Q extends SystemOneQuestions = SystemOneQuestions>(name: string, options: SystemOneJudgeOptions<A, S, Q>): JudgeConfig<A, S>;
|
|
46
|
+
export interface SystemOneAnalystOptions<I, Q extends SystemOneQuestions = SystemOneQuestions> extends SystemOneEvaluatorOptions {
|
|
47
|
+
id: string;
|
|
48
|
+
description: string;
|
|
49
|
+
inputKind: Analyst<I>['inputKind'];
|
|
50
|
+
model: string;
|
|
51
|
+
version: string;
|
|
52
|
+
questions: Q | ((input: I, context: AnalystContext) => Q | Promise<Q>);
|
|
53
|
+
renderState: (input: I, context: AnalystContext) => SystemOneState | Promise<SystemOneState>;
|
|
54
|
+
findings: (value: SystemOneResult<Q>, input: I, context: AnalystContext) => AnalystFinding[] | Promise<AnalystFinding[]>;
|
|
55
|
+
record?: (result: EvaluationResult<SystemOneResult<Q>>, input: I, context: AnalystContext) => void | Promise<void>;
|
|
56
|
+
}
|
|
57
|
+
export declare function systemOneAnalyst<I, Q extends SystemOneQuestions = SystemOneQuestions>(options: SystemOneAnalystOptions<I, Q>): Analyst<I>;
|
|
58
|
+
//#endregion
|
|
59
|
+
export { EvaluationAnalystOptions, EvaluationContext, EvaluationJudgeOptions, EvaluationResult, Evaluator, EvaluatorOptions, PreparedSystemOneReview, SystemOneAnswer, SystemOneAnswerFor, SystemOneDescription, SystemOneJson, SystemOneQuestion, SystemOneQuestions, SystemOneRecordedQuestion, SystemOneRecordedQuestions, SystemOneRecordedRequest, SystemOneRequest, SystemOneResponseError, SystemOneResult, SystemOneResultProfile, SystemOneReviewAssessment, SystemOneReviewCheck, SystemOneReviewInput, SystemOneReviewReport, SystemOneScoreCriteria, SystemOneState, asAnalyst, asJudge, assessSystemOneReview, createEvaluator, normalizeSystemOneResult, parseSystemOneQuestions, parseSystemOneRecordedRequest, parseSystemOneRequest, parseSystemOneResult, prepareSystemOneReview, systemOneConfidence, systemOneReviewFindings, systemOneUsage };
|
|
60
|
+
//# sourceMappingURL=systemone.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"systemone.d.ts","names":[],"sources":["../src/systemone.ts"],"mappings":";;;;;;;YAyBY,oBAAoB,iBAAiB;iBAChC;EACf,UAAU;EACV,aAAa,iBAAiB;EAC9B,gBAAgB,iBAAiB;EACjC,UAAU;EACV,WAAW,sBAAsB;EACjC,oBAAoB,OAAO,UAAU;;EAErC,eAAe,mBAAmB;;;wBAIpB,mBAAmB,SAAS,mCA2BtB,UAAU,oBAAkB,SACrC,iBAAiB,IAAE,UAClB,sBACT,QAAQ,iBAAiB,gBAAgB;KAUzC,MAAM,GAAG,UAAU;EAAc,UAAU;EAAG,UAAU;;KACxD,eAAe,GAAG,UAAU,sBAAsB,MAAM,OAAO,MAAM,IAAI,QAAQ;iBACrE,yBAAyB;EACxC;EACA;EACA,WAAW;;iBAEI,sBACf,GACA,UAAU,WAAW,UACrB,UAAU,qBAAqB,4BACvB;EACR;EACA;EACA,WAAW,eAAe,MAAM,GAAG,IAAI;EACvC,cACE,OAAO,MAAM,GAAG,IAChB,SAAS,sBACN,iBAAiB,QAAQ;EAC9B,aAAa,YAAY,GAAG;EAC5B,UAAU;EACV,OAAO,OAAO,gBAAgB,IAAI,OAAO,MAAM,GAAG,OAAO,aAAa,QAAQ;EAC9E,UACE,QAAQ,iBAAiB,gBAAgB,KACzC,OAAO,MAAM,GAAG,cACN;EACZ,aAAa,UAAU;;;wBA0BT,yBACd,QAAQ,iBACR,UAAU,yBACT;;wBAgCa,eACd,GACA,UAAU,WAAW,UACrB,UAAU,qBAAqB,oBAC/B,cAAc,SAAS,sBAAsB,GAAG,GAAG,KAAK,YAAY,GAAG;iBAqDxD,wBAAwB,GAAG,UAAU,qBAAqB,4BACjE;EACR;EACA;EACA,WAAW,QAAQ;EACnB;EACA;EACA,WAAW,MAAM,OAAO,GAAG,SAAS,mBAAmB,IAAI,QAAQ;EACnE,cAAc,OAAO,GAAG,SAAS,mBAAmB,iBAAiB,QAAQ;EAC7E,WACE,OAAO,gBAAgB,IACvB,OAAO,GACP,SAAS,mBACN,mBAAmB,QAAQ;EAChC,UACE,QAAQ,iBAAiB,gBAAgB,KACzC,OAAO,GACP,SAAS,0BACC;;wBAGE,iBAAiB,GAAG,UAAU,qBAAqB,oBACjE,SAAS,wBAAwB,GAAG,KACnC,QAAQ"}
|
|
@@ -2,11 +2,11 @@ import { o as jsonDocument } from "./canonical-BsZVgBhW.js";
|
|
|
2
2
|
import { s as weightedComposite } from "./descriptive-CnU8Yq29.js";
|
|
3
3
|
import { n as contentHash } from "./verdict-cache-BQEsVy9n.js";
|
|
4
4
|
import { asAnalyst, asJudge, createEvaluator } from "./evaluation.js";
|
|
5
|
-
import {
|
|
6
|
-
import { n as
|
|
7
|
-
//#region src/
|
|
5
|
+
import { SystemOneResponseError, normalizeSystemOneResult, parseSystemOneQuestions, parseSystemOneRecordedRequest, parseSystemOneRequest, parseSystemOneResult, systemOneConfidence, systemOneUsage } from "./systemone-protocol.js";
|
|
6
|
+
import { n as prepareSystemOneReview, r as systemOneReviewFindings, t as assessSystemOneReview } from "./systemone-review-DTClwgmE.js";
|
|
7
|
+
//#region src/systemone.ts
|
|
8
8
|
/** Configure transport once; supply native state, questions, and model on every invocation. */
|
|
9
|
-
function
|
|
9
|
+
function systemOneEvaluator(options) {
|
|
10
10
|
const config = {
|
|
11
11
|
...options,
|
|
12
12
|
pricing: options.pricing && { ...options.pricing }
|
|
@@ -18,7 +18,7 @@ function jevEvaluator(options) {
|
|
|
18
18
|
maximumCharge: config.maximumCharge,
|
|
19
19
|
receiptFromError: config.receiptFromError,
|
|
20
20
|
receipt: config.receipt ?? ((raw) => {
|
|
21
|
-
const usage =
|
|
21
|
+
const usage = systemOneUsage(raw);
|
|
22
22
|
const { model } = raw;
|
|
23
23
|
return {
|
|
24
24
|
model,
|
|
@@ -28,13 +28,13 @@ function jevEvaluator(options) {
|
|
|
28
28
|
};
|
|
29
29
|
}),
|
|
30
30
|
validate: (raw, request) => {
|
|
31
|
-
const result =
|
|
31
|
+
const result = parseSystemOneResult(raw, request);
|
|
32
32
|
if (config.acceptModel && !config.acceptModel(request.model, result.model)) throw new Error("Evaluation served a model rejected by the caller policy");
|
|
33
33
|
}
|
|
34
34
|
});
|
|
35
35
|
return async (request, context) => {
|
|
36
36
|
context?.signal?.throwIfAborted();
|
|
37
|
-
|
|
37
|
+
parseSystemOneRequest(request);
|
|
38
38
|
const snapshot = structuredClone(request);
|
|
39
39
|
const result = await run(snapshot, context);
|
|
40
40
|
return {
|
|
@@ -54,7 +54,7 @@ function validateWeights(weights, keys) {
|
|
|
54
54
|
if (Object.entries(weights).some(([key, weight]) => !keys.includes(key) || !Number.isFinite(weight) || weight < 0) || Object.values(weights).reduce((sum, weight) => sum + weight, 0) <= 0) throw new TypeError("Invalid judge weights");
|
|
55
55
|
}
|
|
56
56
|
/** Optional convention. Choice utilities and non-uniform scales belong in an explicit map. */
|
|
57
|
-
function
|
|
57
|
+
function normalizedSystemOneScore(result, weights) {
|
|
58
58
|
const dimensions = Object.create(null);
|
|
59
59
|
const distribution = Object.create(null);
|
|
60
60
|
for (const [key, answer] of Object.entries(result.answers)) {
|
|
@@ -91,14 +91,14 @@ function normalizedJevScore(result, weights) {
|
|
|
91
91
|
};
|
|
92
92
|
}
|
|
93
93
|
/** Convenience around the public evaluator. No domain questions, thresholds, or prose. */
|
|
94
|
-
function
|
|
94
|
+
function systemOneJudge(name, options) {
|
|
95
95
|
const config = { ...options };
|
|
96
96
|
const source = typeof options.questions === "function" ? options.questions : structuredClone(options.questions);
|
|
97
97
|
const weights = options.weights && { ...options.weights };
|
|
98
98
|
if (!name.trim() || !options.version.trim()) throw new TypeError("Judge name and version are required");
|
|
99
99
|
if ((typeof source === "function" || config.map) && !config.dimensions) throw new TypeError("Dynamic questions and custom mappings require stable judge dimensions");
|
|
100
100
|
if (!config.map && typeof source !== "function") {
|
|
101
|
-
|
|
101
|
+
parseSystemOneQuestions(source);
|
|
102
102
|
if (Object.values(source).some((question) => question.type === "choice")) throw new TypeError("Choice scoring requires an explicit map; labels have no numeric ordering");
|
|
103
103
|
if (weights) validateWeights(weights, Object.keys(source));
|
|
104
104
|
}
|
|
@@ -106,7 +106,7 @@ function jevJudge(name, options) {
|
|
|
106
106
|
key,
|
|
107
107
|
description: typeof question.instructions === "string" ? question.instructions : key
|
|
108
108
|
}));
|
|
109
|
-
const evaluate =
|
|
109
|
+
const evaluate = systemOneEvaluator(config);
|
|
110
110
|
return asJudge({
|
|
111
111
|
name,
|
|
112
112
|
version: contentHash({
|
|
@@ -123,15 +123,15 @@ function jevJudge(name, options) {
|
|
|
123
123
|
state: await config.renderState(input, context),
|
|
124
124
|
questions: typeof source === "function" ? await source(input) : source
|
|
125
125
|
}, context),
|
|
126
|
-
map: config.map ?? ((value) =>
|
|
126
|
+
map: config.map ?? ((value) => normalizedSystemOneScore(value, weights)),
|
|
127
127
|
record: config.record
|
|
128
128
|
});
|
|
129
129
|
}
|
|
130
|
-
function
|
|
130
|
+
function systemOneAnalyst(options) {
|
|
131
131
|
const config = { ...options };
|
|
132
132
|
const source = typeof options.questions === "function" ? options.questions : structuredClone(options.questions);
|
|
133
133
|
if (!config.id.trim() || !config.version.trim() || !config.description.trim()) throw new TypeError("Analyst id, version, and description are required");
|
|
134
|
-
const evaluate =
|
|
134
|
+
const evaluate = systemOneEvaluator(config);
|
|
135
135
|
return asAnalyst({
|
|
136
136
|
id: config.id,
|
|
137
137
|
description: config.description,
|
|
@@ -155,6 +155,6 @@ function jevAnalyst(options) {
|
|
|
155
155
|
});
|
|
156
156
|
}
|
|
157
157
|
//#endregion
|
|
158
|
-
export {
|
|
158
|
+
export { SystemOneResponseError, asAnalyst, asJudge, assessSystemOneReview, createEvaluator, normalizeSystemOneResult, normalizedSystemOneScore, parseSystemOneQuestions, parseSystemOneRecordedRequest, parseSystemOneRequest, parseSystemOneResult, prepareSystemOneReview, systemOneAnalyst, systemOneConfidence, systemOneEvaluator, systemOneJudge, systemOneReviewFindings, systemOneUsage };
|
|
159
159
|
|
|
160
|
-
//# sourceMappingURL=
|
|
160
|
+
//# sourceMappingURL=systemone.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"systemone.js","names":[],"sources":["../src/systemone.ts"],"sourcesContent":["import type { Analyst, AnalystContext, AnalystFinding } from './analyst/types'\nimport type { JudgeConfig, JudgeScore, Scenario } from './campaign/types'\nimport type { CostReceiptInput, CustomTokenPricing } from './cost-ledger'\nimport type { EvaluationContext, EvaluationResult, EvaluatorOptions } from './evaluation'\nimport { asAnalyst, asJudge, createEvaluator } from './evaluation'\nimport { jsonDocument } from './ledger-core/canonical'\nimport { weightedComposite } from './statistics'\nimport type {\n SystemOneQuestions,\n SystemOneRequest,\n SystemOneResult,\n SystemOneState,\n} from './systemone-protocol'\nimport {\n parseSystemOneQuestions,\n parseSystemOneRequest,\n parseSystemOneResult,\n systemOneUsage,\n} from './systemone-protocol'\nimport { contentHash } from './verdict-cache'\n\nexport * from './evaluation'\nexport * from './systemone-protocol'\nexport * from './systemone-review'\n\nexport type SystemOneEvaluate = EvaluatorOptions<SystemOneRequest, unknown>['execute']\nexport interface SystemOneEvaluatorOptions {\n evaluate: SystemOneEvaluate\n costLedger?: EvaluatorOptions<SystemOneRequest, unknown>['costLedger']\n maximumCharge?: EvaluatorOptions<SystemOneRequest, unknown>['maximumCharge']\n pricing?: CustomTokenPricing\n receipt?: (response: unknown) => CostReceiptInput\n receiptFromError?: (error: Error) => CostReceiptInput | undefined\n /** Optional deployment policy; the protocol adapter does not hardcode model aliases. */\n acceptModel?: (requested: string, served: string) => boolean\n}\n\n/** Configure transport once; supply native state, questions, and model on every invocation. */\nexport function systemOneEvaluator(options: SystemOneEvaluatorOptions) {\n const config = { ...options, pricing: options.pricing && { ...options.pricing } }\n const run = createEvaluator<SystemOneRequest, unknown>({\n execute: (request, context) => config.evaluate(structuredClone(request), context),\n model: (request) => request.model,\n costLedger: config.costLedger,\n maximumCharge: config.maximumCharge,\n receiptFromError: config.receiptFromError,\n receipt:\n config.receipt ??\n ((raw) => {\n const usage = systemOneUsage(raw)\n const { model } = raw as SystemOneResult\n return {\n model,\n inputTokens: usage.input_tokens,\n outputTokens: usage.output_tokens,\n customTokenPricing: config.pricing,\n }\n }),\n validate: (raw, request) => {\n const result = parseSystemOneResult(raw, request)\n if (config.acceptModel && !config.acceptModel(request.model, result.model)) {\n throw new Error('Evaluation served a model rejected by the caller policy')\n }\n },\n })\n return async <const Q extends SystemOneQuestions>(\n request: SystemOneRequest<Q>,\n context?: EvaluationContext,\n ): Promise<EvaluationResult<SystemOneResult<Q>>> => {\n context?.signal?.throwIfAborted()\n parseSystemOneRequest(request)\n const snapshot = structuredClone(request)\n const result = await run(snapshot, context)\n // The validator checked these exact questions after recording the paid work.\n return { ...result, value: result.value as SystemOneResult<Q> }\n }\n}\n\ntype Input<A, S extends Scenario> = { artifact: A; scenario: S }\ntype QuestionSource<I, Q extends SystemOneQuestions> = Q | ((input: I) => Q | Promise<Q>)\nexport interface SystemOneOptions extends SystemOneEvaluatorOptions {\n model: string\n version: string\n questions: SystemOneQuestions\n}\nexport interface SystemOneJudgeOptions<\n A,\n S extends Scenario = Scenario,\n Q extends SystemOneQuestions = SystemOneQuestions,\n> extends SystemOneEvaluatorOptions {\n model: string\n version: string\n questions: QuestionSource<Input<A, S>, Q>\n renderState: (\n input: Input<A, S>,\n context: EvaluationContext,\n ) => SystemOneState | Promise<SystemOneState>\n dimensions?: JudgeConfig<A, S>['dimensions']\n weights?: Record<string, number>\n map?: (value: SystemOneResult<Q>, input: Input<A, S>) => JudgeScore | Promise<JudgeScore>\n record?: (\n result: EvaluationResult<SystemOneResult<Q>>,\n input: Input<A, S>,\n ) => void | Promise<void>\n appliesTo?: (scenario: S) => boolean\n}\n\n/** Canonical identity also retains native question/alternative order; dispatch stays untouched. */\nfunction questionIdentity(questions: SystemOneQuestions) {\n return {\n document: jsonDocument(questions),\n order: Object.entries(questions).map(([name, question]) => [\n name,\n question.type === 'choice' ? Object.keys(question.criteria) : null,\n ]),\n }\n}\n\nfunction validateWeights(weights: Record<string, number>, keys: string[]): void {\n if (\n Object.entries(weights).some(\n ([key, weight]) => !keys.includes(key) || !Number.isFinite(weight) || weight < 0,\n ) ||\n Object.values(weights).reduce((sum, weight) => sum + weight, 0) <= 0\n ) {\n throw new TypeError('Invalid judge weights')\n }\n}\n\n/** Optional convention. Choice utilities and non-uniform scales belong in an explicit map. */\nexport function normalizedSystemOneScore(\n result: SystemOneResult,\n weights?: Record<string, number>,\n): JudgeScore {\n const dimensions: Record<string, number> = Object.create(null)\n const distribution: NonNullable<JudgeScore['distribution']> = Object.create(null)\n for (const [key, answer] of Object.entries(result.answers)) {\n if (answer.type === 'choice') throw new TypeError('Choice scoring requires an explicit map')\n if (answer.type === 'noul') {\n dimensions[key] = answer.noul\n distribution[key] = [\n { score: 0, probability: 1 - answer.noul },\n { score: 1, probability: answer.noul },\n ]\n } else {\n const scale = Object.keys(answer.legend).length - 1\n dimensions[key] = answer.score / scale\n distribution[key] = Object.entries(answer.probabilities).map(([level, probability]) => ({\n score: Number(level) / scale,\n probability,\n }))\n }\n }\n const effective = weights ?? Object.fromEntries(Object.keys(dimensions).map((key) => [key, 1]))\n validateWeights(effective, Object.keys(dimensions))\n return {\n dimensions,\n composite: weightedComposite({ dims: dimensions, weights: effective }).composite,\n notes: '',\n scoringMethod: 'expectation',\n distribution,\n }\n}\n\n/** Convenience around the public evaluator. No domain questions, thresholds, or prose. */\nexport function systemOneJudge<\n A,\n S extends Scenario = Scenario,\n Q extends SystemOneQuestions = SystemOneQuestions,\n>(name: string, options: SystemOneJudgeOptions<A, S, Q>): JudgeConfig<A, S> {\n const config = { ...options }\n const source =\n typeof options.questions === 'function' ? options.questions : structuredClone(options.questions)\n const weights = options.weights && { ...options.weights }\n if (!name.trim() || !options.version.trim()) {\n throw new TypeError('Judge name and version are required')\n }\n if ((typeof source === 'function' || config.map) && !config.dimensions) {\n throw new TypeError('Dynamic questions and custom mappings require stable judge dimensions')\n }\n if (!config.map && typeof source !== 'function') {\n parseSystemOneQuestions(source)\n if (Object.values(source).some((question) => question.type === 'choice')) {\n throw new TypeError(\n 'Choice scoring requires an explicit map; labels have no numeric ordering',\n )\n }\n if (weights) validateWeights(weights, Object.keys(source))\n }\n const dimensions =\n config.dimensions ??\n Object.entries(source).map(([key, question]) => ({\n key,\n description: typeof question.instructions === 'string' ? question.instructions : key,\n }))\n const evaluate = systemOneEvaluator(config)\n return asJudge<A, S, SystemOneResult<Q>>({\n name,\n version: contentHash({\n model: config.model,\n version: config.version,\n questions: typeof source === 'function' ? 'dynamic' : questionIdentity(source),\n dimensions,\n // Hash the JSON document: optional undefined fields are absent on the wire.\n ...(weights ? { weights } : {}),\n }),\n dimensions,\n appliesTo: config.appliesTo,\n evaluate: async (input, context = {}) =>\n evaluate(\n {\n model: config.model,\n state: await config.renderState(input, context),\n questions: typeof source === 'function' ? await source(input) : source,\n },\n context,\n ),\n map: config.map ?? ((value) => normalizedSystemOneScore(value, weights)),\n record: config.record,\n })\n}\n\nexport interface SystemOneAnalystOptions<I, Q extends SystemOneQuestions = SystemOneQuestions>\n extends SystemOneEvaluatorOptions {\n id: string\n description: string\n inputKind: Analyst<I>['inputKind']\n model: string\n version: string\n questions: Q | ((input: I, context: AnalystContext) => Q | Promise<Q>)\n renderState: (input: I, context: AnalystContext) => SystemOneState | Promise<SystemOneState>\n findings: (\n value: SystemOneResult<Q>,\n input: I,\n context: AnalystContext,\n ) => AnalystFinding[] | Promise<AnalystFinding[]>\n record?: (\n result: EvaluationResult<SystemOneResult<Q>>,\n input: I,\n context: AnalystContext,\n ) => void | Promise<void>\n}\n\nexport function systemOneAnalyst<I, Q extends SystemOneQuestions = SystemOneQuestions>(\n options: SystemOneAnalystOptions<I, Q>,\n): Analyst<I> {\n const config = { ...options }\n const source =\n typeof options.questions === 'function' ? options.questions : structuredClone(options.questions)\n if (!config.id.trim() || !config.version.trim() || !config.description.trim()) {\n throw new TypeError('Analyst id, version, and description are required')\n }\n const evaluate = systemOneEvaluator(config)\n return asAnalyst<I, SystemOneResult<Q>>({\n id: config.id,\n description: config.description,\n inputKind: config.inputKind,\n version: contentHash({\n model: config.model,\n version: config.version,\n questions: typeof source === 'function' ? 'dynamic' : questionIdentity(source),\n }),\n cost: { kind: 'llm', models: [config.model] },\n evaluate: async (input, context, analystContext) =>\n evaluate(\n {\n model: config.model,\n state: await config.renderState(input, analystContext),\n questions: typeof source === 'function' ? await source(input, analystContext) : source,\n },\n context,\n ),\n map: config.findings,\n record: config.record,\n })\n}\n"],"mappings":";;;;;;;;AAsCA,SAAgB,mBAAmB,SAAoC;CACrE,MAAM,SAAS;EAAE,GAAG;EAAS,SAAS,QAAQ,WAAW,EAAE,GAAG,QAAQ,QAAQ;CAAE;CAChF,MAAM,MAAM,gBAA2C;EACrD,UAAU,SAAS,YAAY,OAAO,SAAS,gBAAgB,OAAO,GAAG,OAAO;EAChF,QAAQ,YAAY,QAAQ;EAC5B,YAAY,OAAO;EACnB,eAAe,OAAO;EACtB,kBAAkB,OAAO;EACzB,SACE,OAAO,aACL,QAAQ;GACR,MAAM,QAAQ,eAAe,GAAG;GAChC,MAAM,EAAE,UAAU;GAClB,OAAO;IACL;IACA,aAAa,MAAM;IACnB,cAAc,MAAM;IACpB,oBAAoB,OAAO;GAC7B;EACF;EACF,WAAW,KAAK,YAAY;GAC1B,MAAM,SAAS,qBAAqB,KAAK,OAAO;GAChD,IAAI,OAAO,eAAe,CAAC,OAAO,YAAY,QAAQ,OAAO,OAAO,KAAK,GACvE,MAAM,IAAI,MAAM,yDAAyD;EAE7E;CACF,CAAC;CACD,OAAO,OACL,SACA,YACkD;EAClD,SAAS,QAAQ,eAAe;EAChC,sBAAsB,OAAO;EAC7B,MAAM,WAAW,gBAAgB,OAAO;EACxC,MAAM,SAAS,MAAM,IAAI,UAAU,OAAO;EAE1C,OAAO;GAAE,GAAG;GAAQ,OAAO,OAAO;EAA4B;CAChE;AACF;;AAgCA,SAAS,iBAAiB,WAA+B;CACvD,OAAO;EACL,UAAU,aAAa,SAAS;EAChC,OAAO,OAAO,QAAQ,SAAS,CAAC,CAAC,KAAK,CAAC,MAAM,cAAc,CACzD,MACA,SAAS,SAAS,WAAW,OAAO,KAAK,SAAS,QAAQ,IAAI,IAChE,CAAC;CACH;AACF;AAEA,SAAS,gBAAgB,SAAiC,MAAsB;CAC9E,IACE,OAAO,QAAQ,OAAO,CAAC,CAAC,MACrB,CAAC,KAAK,YAAY,CAAC,KAAK,SAAS,GAAG,KAAK,CAAC,OAAO,SAAS,MAAM,KAAK,SAAS,CACjF,KACA,OAAO,OAAO,OAAO,CAAC,CAAC,QAAQ,KAAK,WAAW,MAAM,QAAQ,CAAC,KAAK,GAEnE,MAAM,IAAI,UAAU,uBAAuB;AAE/C;;AAGA,SAAgB,yBACd,QACA,SACY;CACZ,MAAM,aAAqC,OAAO,OAAO,IAAI;CAC7D,MAAM,eAAwD,OAAO,OAAO,IAAI;CAChF,KAAK,MAAM,CAAC,KAAK,WAAW,OAAO,QAAQ,OAAO,OAAO,GAAG;EAC1D,IAAI,OAAO,SAAS,UAAU,MAAM,IAAI,UAAU,yCAAyC;EAC3F,IAAI,OAAO,SAAS,QAAQ;GAC1B,WAAW,OAAO,OAAO;GACzB,aAAa,OAAO,CAClB;IAAE,OAAO;IAAG,aAAa,IAAI,OAAO;GAAK,GACzC;IAAE,OAAO;IAAG,aAAa,OAAO;GAAK,CACvC;EACF,OAAO;GACL,MAAM,QAAQ,OAAO,KAAK,OAAO,MAAM,CAAC,CAAC,SAAS;GAClD,WAAW,OAAO,OAAO,QAAQ;GACjC,aAAa,OAAO,OAAO,QAAQ,OAAO,aAAa,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB;IACtF,OAAO,OAAO,KAAK,IAAI;IACvB;GACF,EAAE;EACJ;CACF;CACA,MAAM,YAAY,WAAW,OAAO,YAAY,OAAO,KAAK,UAAU,CAAC,CAAC,KAAK,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;CAC9F,gBAAgB,WAAW,OAAO,KAAK,UAAU,CAAC;CAClD,OAAO;EACL;EACA,WAAW,kBAAkB;GAAE,MAAM;GAAY,SAAS;EAAU,CAAC,CAAC,CAAC;EACvE,OAAO;EACP,eAAe;EACf;CACF;AACF;;AAGA,SAAgB,eAId,MAAc,SAA4D;CAC1E,MAAM,SAAS,EAAE,GAAG,QAAQ;CAC5B,MAAM,SACJ,OAAO,QAAQ,cAAc,aAAa,QAAQ,YAAY,gBAAgB,QAAQ,SAAS;CACjG,MAAM,UAAU,QAAQ,WAAW,EAAE,GAAG,QAAQ,QAAQ;CACxD,IAAI,CAAC,KAAK,KAAK,KAAK,CAAC,QAAQ,QAAQ,KAAK,GACxC,MAAM,IAAI,UAAU,qCAAqC;CAE3D,KAAK,OAAO,WAAW,cAAc,OAAO,QAAQ,CAAC,OAAO,YAC1D,MAAM,IAAI,UAAU,uEAAuE;CAE7F,IAAI,CAAC,OAAO,OAAO,OAAO,WAAW,YAAY;EAC/C,wBAAwB,MAAM;EAC9B,IAAI,OAAO,OAAO,MAAM,CAAC,CAAC,MAAM,aAAa,SAAS,SAAS,QAAQ,GACrE,MAAM,IAAI,UACR,0EACF;EAEF,IAAI,SAAS,gBAAgB,SAAS,OAAO,KAAK,MAAM,CAAC;CAC3D;CACA,MAAM,aACJ,OAAO,cACP,OAAO,QAAQ,MAAM,CAAC,CAAC,KAAK,CAAC,KAAK,eAAe;EAC/C;EACA,aAAa,OAAO,SAAS,iBAAiB,WAAW,SAAS,eAAe;CACnF,EAAE;CACJ,MAAM,WAAW,mBAAmB,MAAM;CAC1C,OAAO,QAAkC;EACvC;EACA,SAAS,YAAY;GACnB,OAAO,OAAO;GACd,SAAS,OAAO;GAChB,WAAW,OAAO,WAAW,aAAa,YAAY,iBAAiB,MAAM;GAC7E;GAEA,GAAI,UAAU,EAAE,QAAQ,IAAI,CAAC;EAC/B,CAAC;EACD;EACA,WAAW,OAAO;EAClB,UAAU,OAAO,OAAO,UAAU,CAAC,MACjC,SACE;GACE,OAAO,OAAO;GACd,OAAO,MAAM,OAAO,YAAY,OAAO,OAAO;GAC9C,WAAW,OAAO,WAAW,aAAa,MAAM,OAAO,KAAK,IAAI;EAClE,GACA,OACF;EACF,KAAK,OAAO,SAAS,UAAU,yBAAyB,OAAO,OAAO;EACtE,QAAQ,OAAO;CACjB,CAAC;AACH;AAuBA,SAAgB,iBACd,SACY;CACZ,MAAM,SAAS,EAAE,GAAG,QAAQ;CAC5B,MAAM,SACJ,OAAO,QAAQ,cAAc,aAAa,QAAQ,YAAY,gBAAgB,QAAQ,SAAS;CACjG,IAAI,CAAC,OAAO,GAAG,KAAK,KAAK,CAAC,OAAO,QAAQ,KAAK,KAAK,CAAC,OAAO,YAAY,KAAK,GAC1E,MAAM,IAAI,UAAU,mDAAmD;CAEzE,MAAM,WAAW,mBAAmB,MAAM;CAC1C,OAAO,UAAiC;EACtC,IAAI,OAAO;EACX,aAAa,OAAO;EACpB,WAAW,OAAO;EAClB,SAAS,YAAY;GACnB,OAAO,OAAO;GACd,SAAS,OAAO;GAChB,WAAW,OAAO,WAAW,aAAa,YAAY,iBAAiB,MAAM;EAC/E,CAAC;EACD,MAAM;GAAE,MAAM;GAAO,QAAQ,CAAC,OAAO,KAAK;EAAE;EAC5C,UAAU,OAAO,OAAO,SAAS,mBAC/B,SACE;GACE,OAAO,OAAO;GACd,OAAO,MAAM,OAAO,YAAY,OAAO,cAAc;GACrD,WAAW,OAAO,WAAW,aAAa,MAAM,OAAO,OAAO,cAAc,IAAI;EAClF,GACA,OACF;EACF,KAAK,OAAO;EACZ,QAAQ,OAAO;CACjB,CAAC;AACH"}
|
|
@@ -3893,12 +3893,19 @@ function fixedOrDash(value) {
|
|
|
3893
3893
|
}
|
|
3894
3894
|
//#endregion
|
|
3895
3895
|
//#region src/search/lenses/task-matrix.ts
|
|
3896
|
+
const TASK_MATRIX_NODES = 200;
|
|
3896
3897
|
const SPECIALIST_GAIN_METHOD = `per unit cluster, the best node's mean over the cluster's units minus the mean of the node means, in the objective's direction and the metric's units, among nodes not decided invalid that scored every unit of the cluster; reported from 2 such nodes and ${INSUFFICIENT_FROM} units; a point value with no interval: the largest of several noisy means exceeds their mean even when no node specializes, so read it against the spread of the node means`;
|
|
3897
3898
|
function taskMatrix(state, options = {}) {
|
|
3898
3899
|
const split = options.split ?? rankingSplit(state);
|
|
3899
3900
|
const direction = state.header?.objective.direction ?? "maximize";
|
|
3900
3901
|
const sign = direction === "maximize" ? 1 : -1;
|
|
3901
|
-
const
|
|
3902
|
+
const allNodeIds = state.nodeIds();
|
|
3903
|
+
const nodeIds = evenSample(allNodeIds, TASK_MATRIX_NODES);
|
|
3904
|
+
const sampling = nodeIds.length < allNodeIds.length ? {
|
|
3905
|
+
nodesClustered: nodeIds.length,
|
|
3906
|
+
nodesTotal: allNodeIds.length,
|
|
3907
|
+
cap: TASK_MATRIX_NODES
|
|
3908
|
+
} : null;
|
|
3902
3909
|
const invalid = new Set(state.nodes().filter((node) => node.status === "invalid").map((node) => node.nodeId));
|
|
3903
3910
|
const byNodeUnit = /* @__PURE__ */ new Map();
|
|
3904
3911
|
const unitSet = /* @__PURE__ */ new Set();
|
|
@@ -3970,7 +3977,8 @@ function taskMatrix(state, options = {}) {
|
|
|
3970
3977
|
data: {
|
|
3971
3978
|
split,
|
|
3972
3979
|
direction,
|
|
3973
|
-
method: SPECIALIST_GAIN_METHOD,
|
|
3980
|
+
method: sampling ? `${SPECIALIST_GAIN_METHOD}; node clustering, unit clustering, and specialist gains use ${sampling.nodesClustered} of ${sampling.nodesTotal} nodes, evenly sampled in registration order (cap ${sampling.cap})` : SPECIALIST_GAIN_METHOD,
|
|
3981
|
+
...sampling ? { sampling } : {},
|
|
3974
3982
|
nodeIds: orderedNodeIds,
|
|
3975
3983
|
unitIds: orderedUnitIds,
|
|
3976
3984
|
nodeClusters,
|
|
@@ -4104,4 +4112,4 @@ function tree(state) {
|
|
|
4104
4112
|
//#endregion
|
|
4105
4113
|
export { editCreditText as A, draftOnPlateau as C, EDIT_CREDIT_ESTIMATOR as D, front as E, EDIT_CREDIT_SIGNAL as O, vectorEmbedding as S, screenedNodes as T, lineageEdits as _, skillCalibration as a, surfaceDigestEdits as b, META_SEARCH_SIGNAL as c, objectiveKey as d, renderMetaSearchText as f, lineEditDistance as g, landscape as h, nextUnitExtension as i, editCredit as k, metaSearch as l, formatLandscape as m, taskMatrix as n, skillManifold as o, searchPolicyGenome as p, formatSkillManifold as r, META_SEARCH_SCORE_SOURCE as s, tree as t, metaSearchScore as u, nearestNeighbours as v, searchPlateau as w, surfaceTextEdits as x, profileTextLines as y };
|
|
4106
4114
|
|
|
4107
|
-
//# sourceMappingURL=tree-
|
|
4115
|
+
//# sourceMappingURL=tree-CN8mlzUO.js.map
|