@tangle-network/agent-eval 0.199.0 → 0.200.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -1
- package/dist/adapters/http.d.ts +2 -2
- package/dist/analyst/index.d.ts +9 -69
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +1 -1
- package/dist/{benchmark-command-CoBwYvau.js → benchmark-command-EBmuMUBo.js} +5 -3
- package/dist/benchmark-command-EBmuMUBo.js.map +1 -0
- package/dist/benchmarks/index.d.ts +4 -4
- package/dist/benchmarks/index.js +2 -2
- package/dist/campaign/index.d.ts +4 -4
- package/dist/campaign/index.js +6 -6
- package/dist/{campaign-BIqhoN9r.js → campaign-L391bmir.js} +7 -7
- package/dist/{campaign-BIqhoN9r.js.map → campaign-L391bmir.js.map} +1 -1
- package/dist/cli.js +11 -9
- package/dist/cli.js.map +1 -1
- package/dist/contract/index.d.ts +7 -7
- package/dist/contract/index.js +3 -3
- package/dist/{default-registry-oK9hHfX_.d.ts → default-registry-D1rNS2yB.d.ts} +3 -3
- package/dist/{default-registry-oK9hHfX_.d.ts.map → default-registry-D1rNS2yB.d.ts.map} +1 -1
- package/dist/{define-agent-eval-Bgwt5ZcC.d.ts → define-agent-eval-1YuvJpOs.d.ts} +5 -5
- package/dist/{define-agent-eval-Bgwt5ZcC.d.ts.map → define-agent-eval-1YuvJpOs.d.ts.map} +1 -1
- package/dist/{define-agent-eval-BRCLti7N.js → define-agent-eval-7q_UwsTt.js} +3 -3
- package/dist/{define-agent-eval-BRCLti7N.js.map → define-agent-eval-7q_UwsTt.js.map} +1 -1
- package/dist/{engine-Dc996kAg.d.ts → engine-DlKIzDKo.d.ts} +2 -2
- package/dist/{engine-Dc996kAg.d.ts.map → engine-DlKIzDKo.d.ts.map} +1 -1
- package/dist/evaluation.d.ts +1 -1
- package/dist/experiment/index.d.ts +1 -1
- package/dist/hosted/index.js +1 -1
- package/dist/{index-lbwvO9vJ.d.ts → index-Bi2fLITd.d.ts} +3 -3
- package/dist/{index-lbwvO9vJ.d.ts.map → index-Bi2fLITd.d.ts.map} +1 -1
- package/dist/{index-D4pC7Tvd.d.ts → index-D4pC7Tvd2.d.ts} +1 -1
- package/dist/index-D4pC7Tvd2.d.ts.map +1 -0
- package/dist/{index-OuIM7sJz.d.ts → index-DKmrHA02.d.ts} +4 -4
- package/dist/{index-OuIM7sJz.d.ts.map → index-DKmrHA02.d.ts.map} +1 -1
- package/dist/{index-DNgf5gyG.d.ts → index-DNgf5gyG2.d.ts} +1 -1
- package/dist/index-DNgf5gyG2.d.ts.map +1 -0
- package/dist/{index-J8BB_NYv.d.ts → index-EkRNsLr4.d.ts} +6 -6
- package/dist/{index-J8BB_NYv.d.ts.map → index-EkRNsLr4.d.ts.map} +1 -1
- package/dist/index.d.ts +10 -10
- package/dist/index.js +4 -4
- package/dist/jev-protocol-CpKI7XSu.d.ts +82 -0
- package/dist/jev-protocol-CpKI7XSu.d.ts.map +1 -0
- package/dist/jev-protocol.d.ts +2 -82
- package/dist/jev-review-Cu9xtIu4.js +155 -0
- package/dist/jev-review-Cu9xtIu4.js.map +1 -0
- package/dist/jev-review-D4ynCORp.d.ts +76 -0
- package/dist/jev-review-D4ynCORp.d.ts.map +1 -0
- package/dist/jev.d.ts +3 -74
- package/dist/jev.d.ts.map +1 -1
- package/dist/jev.js +2 -151
- package/dist/jev.js.map +1 -1
- package/dist/{llm-judge-GSlZ0eG8.js → llm-judge-CL6f3utD.js} +4 -4
- package/dist/{llm-judge-GSlZ0eG8.js.map → llm-judge-CL6f3utD.js.map} +1 -1
- package/dist/matrix/index.d.ts +1 -1
- package/dist/{matrix-BnksRaDD.d.ts → matrix-_NLSWsKM.d.ts} +3 -3
- package/dist/{matrix-BnksRaDD.d.ts.map → matrix-_NLSWsKM.d.ts.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/golden/index.js +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/multishot/index.js +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{produced-state-D2ff-Dyo.js → produced-state-DZMW8cuF.js} +2 -2
- package/dist/{produced-state-D2ff-Dyo.js.map → produced-state-DZMW8cuF.js.map} +1 -1
- package/dist/rl.d.ts +279 -2
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +576 -2
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +1 -1
- package/dist/search/index.d.ts +2 -2
- package/dist/search/index.js +2 -2
- package/dist/{search-kernel-CumYAjA3.js → search-kernel-AVgIjXaI.js} +56 -15
- package/dist/search-kernel-AVgIjXaI.js.map +1 -0
- package/dist/{search-kernel-DA2-mS4N.d.ts → search-kernel-C9CXvFR7.d.ts} +6 -4
- package/dist/search-kernel-C9CXvFR7.d.ts.map +1 -0
- package/dist/{search-ledger-BnUytR8X.js → search-ledger-CL0-mXQ8.js} +4 -2
- package/dist/search-ledger-CL0-mXQ8.js.map +1 -0
- package/dist/{search-shipper-Cn8cxcyp.js → search-shipper-Cs6K4km9.js} +2 -2
- package/dist/{search-shipper-Cn8cxcyp.js.map → search-shipper-Cs6K4km9.js.map} +1 -1
- package/dist/{search-summary-dJOaZ3Dw.js → search-summary-DvWk0hb3.js} +2 -2
- package/dist/{search-summary-dJOaZ3Dw.js.map → search-summary-DvWk0hb3.js.map} +1 -1
- package/dist/{skillopt-optimization-method-DDt77V-N.js → skillopt-optimization-method-B2syNik0.js} +3 -3
- package/dist/{skillopt-optimization-method-DDt77V-N.js.map → skillopt-optimization-method-B2syNik0.js.map} +1 -1
- package/dist/{statistical-heldout-CMpxMjlQ.d.ts → statistical-heldout-HswMvzbP.d.ts} +2 -2
- package/dist/{statistical-heldout-CMpxMjlQ.d.ts.map → statistical-heldout-HswMvzbP.d.ts.map} +1 -1
- package/dist/{store-tool-spans-CUwWSXoj.d.ts → store-tool-spans-Dv9x4XYG.d.ts} +3 -3
- package/dist/{store-tool-spans-CUwWSXoj.d.ts.map → store-tool-spans-Dv9x4XYG.d.ts.map} +1 -1
- package/dist/{tool-groups-DVJAR1GE.d.ts → tool-groups-BY1MdSIE.d.ts} +2 -2
- package/dist/tool-groups-BY1MdSIE.d.ts.map +1 -0
- package/dist/trace-questions-DnqdmwyV.d.ts +68 -0
- package/dist/trace-questions-DnqdmwyV.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +1 -1
- package/dist/traces.d.ts +2 -2
- package/dist/{tree-DxGfhISA.js → tree-Ki_gW7z8.js} +25 -26
- package/dist/{tree-DxGfhISA.js.map → tree-Ki_gW7z8.js.map} +1 -1
- package/dist/{types-B3jzCp0p2.js → types-B3jzCp0p.js} +1 -1
- package/dist/types-B3jzCp0p.js.map +1 -0
- package/dist/{types-D94EanDL.d.ts → types-C4hH69a9.d.ts} +2 -2
- package/dist/{types-D94EanDL.d.ts.map → types-C4hH69a9.d.ts.map} +1 -1
- package/dist/{types-EBUav9Bu.d.ts → types-DfvQOe91.d.ts} +4 -1
- package/dist/types-DfvQOe91.d.ts.map +1 -0
- package/docs/jev-behavior-review.md +24 -0
- package/docs/public-api.md +50 -31
- package/docs/search-ledger.md +20 -2
- package/package.json +1 -1
- package/dist/benchmark-command-CoBwYvau.js.map +0 -1
- package/dist/index-D4pC7Tvd.d.ts.map +0 -1
- package/dist/index-DNgf5gyG.d.ts.map +0 -1
- package/dist/jev-protocol.d.ts.map +0 -1
- package/dist/search-kernel-CumYAjA3.js.map +0 -1
- package/dist/search-kernel-DA2-mS4N.d.ts.map +0 -1
- package/dist/search-ledger-BnUytR8X.js.map +0 -1
- package/dist/tool-groups-DVJAR1GE.d.ts.map +0 -1
- package/dist/types-B3jzCp0p2.js.map +0 -1
- package/dist/types-EBUav9Bu.d.ts.map +0 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"jev-review-Cu9xtIu4.js","names":[],"sources":["../src/jev-review.ts"],"sourcesContent":["import { z } from 'zod'\nimport {\n ANALYST_SEVERITIES,\n type AnalystFinding,\n type EvidenceRef,\n makeFinding,\n} from './analyst/types'\nimport type { JevQuestions, JevRequest, JevResult } from './jev-protocol'\nimport { parseJevRequest, parseJevResult } from './jev-protocol'\nimport { canonicalString, hashCanonical, jsonDocument } from './ledger-core/canonical'\nimport { deepFreezeCanonicalJson } from './ledger-core/deep-freeze'\n\nconst name = z.string().trim().min(1)\nconst probability = z.number().finite().gt(0).max(1)\nconst checkSchema = z\n .object({\n claim: name,\n area: name,\n subject: name,\n severity: z.enum(ANALYST_SEVERITIES),\n /** Choice labels or score-level keys, never scores invented from category order. */\n supports: z.array(z.string()).min(1),\n refutes: z.array(z.string()).min(1),\n supportAtLeast: probability,\n refuteAtLeast: probability,\n /** Host-supplied coverage of the question's required evidence, not model self-assessment. */\n coverage: z.enum(['complete', 'partial', 'missing']),\n evidence: z.array(\n z\n .object({\n kind: z.enum(['span', 'event', 'artifact', 'finding', 'metric']),\n uri: name,\n excerpt: z.string().optional(),\n })\n .strict(),\n ),\n })\n .strict()\n .refine((value) => value.supportAtLeast + value.refuteAtLeast > 1, {\n message: 'Support and refutation thresholds must not overlap',\n })\n\n/** Optional interpretation of native choices/scores. Policy and evidence belong to the caller. */\nexport type JevReviewCheck = z.infer<typeof checkSchema>\nexport interface JevReviewInput<Q extends JevQuestions = JevQuestions> {\n version: string\n request: JevRequest<Q>\n checks: { [K in keyof Q]: JevReviewCheck }\n}\n\nexport interface PreparedJevReview<Q extends JevQuestions = JevQuestions>\n extends JevReviewInput<Q> {\n /** Binds questions, option order, context, evidence, thresholds and definition version. */\n digest: string\n}\n\nexport interface JevReviewAssessment {\n question: string\n status: 'supported' | 'refuted' | 'unresolved'\n reason:\n | 'support-threshold'\n | 'refutation-threshold'\n | 'missing-evidence'\n | 'partial-negative-evidence'\n | 'below-threshold'\n | 'ambiguous-thresholds'\n /** Unnormalized marginal mass of caller-designated native alternatives. Not calibrated risk. */\n supportProbability: number\n refutationProbability: number\n unresolvedProbability: number\n modelConfidence: number | null\n check: JevReviewCheck\n}\n\nexport interface JevReviewReport {\n reviewDigest: string\n requestedModel: string\n servedModel: string\n assessments: JevReviewAssessment[]\n}\n\nfunction alternatives(question: JevQuestions[string]): string[] {\n if (question.type === 'noul') return ['true', 'false']\n return question.type === 'choice'\n ? Object.keys(question.criteria)\n : question.criteria.map((_, index) => String(index))\n}\n\n/** Validate before paid admission; make a detached, frozen snapshot without rewriting native JSON. */\nexport function prepareJevReview<const Q extends JevQuestions>(\n input: JevReviewInput<Q>,\n): PreparedJevReview<Q> {\n if (!input.version?.trim()) throw new TypeError('Review version is required')\n parseJevRequest(input.request)\n // Document form drops only optional undefined object fields; canonicalization rejects loss.\n const document = jsonDocument(input)\n canonicalString(document)\n // Canonical bytes are for hashing, not dispatch: option order can affect a classifier.\n const snapshot = structuredClone(document) as JevReviewInput<Q>\n const keys = Object.keys(snapshot.request.questions)\n if (\n !snapshot.checks ||\n Object.keys(snapshot.checks).length !== keys.length ||\n keys.some((key) => !Object.hasOwn(snapshot.checks, key))\n ) {\n throw new TypeError('Review checks must match the exact question names')\n }\n for (const key of keys) {\n const check = checkSchema.parse(snapshot.checks[key])\n const labels = [...check.supports, ...check.refutes]\n const allowed = alternatives(snapshot.request.questions[key]!)\n if (\n new Set(labels).size !== labels.length ||\n labels.some((label) => !allowed.includes(label))\n ) {\n throw new TypeError('Review alternatives must exist and be disjoint')\n }\n if (check.coverage !== 'missing' && check.evidence.length === 0) {\n throw new TypeError('Review evidence references are required for non-missing coverage')\n }\n // Validation must not normalize a label into a different native option.\n if (canonicalString(jsonDocument(check)) !== canonicalString(snapshot.checks[key])) {\n throw new TypeError('Review metadata must not require normalization')\n }\n }\n const order = keys.map((key) => [key, alternatives(snapshot.request.questions[key]!)])\n return deepFreezeCanonicalJson({ ...snapshot, digest: hashCanonical({ snapshot, order }) })\n}\n\n/** Re-score the same retained distribution without new inference or any authorization effect. */\nexport function assessJevReview<Q extends JevQuestions>(\n review: PreparedJevReview<Q>,\n raw: unknown,\n): JevReviewReport {\n const { digest, ...input } = review\n const prepared = prepareJevReview(input)\n if (prepared.digest !== digest) throw new TypeError('Review definition changed after preparation')\n const result = parseJevResult(raw, prepared.request)\n const assessments = Object.entries(prepared.checks).map(([question, check]) => {\n const answer = result.answers[question]!\n const probabilities: Record<string, number> =\n answer.type === 'noul' ? { true: answer.noul, false: 1 - answer.noul } : answer.probabilities\n const sum = (labels: string[]) =>\n labels.reduce((total, label) => total + probabilities[label]!, 0)\n const supportProbability = sum(check.supports)\n const refutationProbability = sum(check.refutes)\n const used = new Set([...check.supports, ...check.refutes])\n const unresolvedProbability = sum(\n Object.keys(probabilities).filter((label) => !used.has(label)),\n )\n let status: JevReviewAssessment['status'] = 'unresolved'\n let reason: JevReviewAssessment['reason'] = 'below-threshold'\n if (supportProbability > 1 || refutationProbability > 1 || unresolvedProbability > 1) {\n throw new TypeError('Review probability mass exceeds one; do not silently normalize it')\n }\n if (check.coverage === 'missing') reason = 'missing-evidence'\n else if (\n supportProbability >= check.supportAtLeast &&\n refutationProbability >= check.refuteAtLeast\n ) {\n reason = 'ambiguous-thresholds'\n } else if (supportProbability >= check.supportAtLeast) {\n status = 'supported'\n reason = 'support-threshold'\n } else if (refutationProbability >= check.refuteAtLeast) {\n if (check.coverage === 'complete') {\n status = 'refuted'\n reason = 'refutation-threshold'\n } else reason = 'partial-negative-evidence'\n }\n return {\n question,\n status,\n reason,\n supportProbability,\n refutationProbability,\n unresolvedProbability,\n modelConfidence: answer.type === 'noul' ? null : answer.confidence,\n check: structuredClone(check),\n }\n })\n return {\n reviewDigest: digest,\n requestedModel: review.request.model,\n servedModel: result.model,\n assessments,\n }\n}\n\n/** Ordinary analyst findings; a negative review is not a global safety certificate. */\nexport function jevReviewFindings<Q extends JevQuestions>(\n review: PreparedJevReview<Q>,\n result: JevResult<Q>,\n options: { analystId: string; producedAt?: string },\n): AnalystFinding[] {\n if (!options.analystId.trim()) throw new TypeError('Analyst id is required')\n const report = assessJevReview(review, result)\n return report.assessments\n .filter((item) => item.status !== 'refuted')\n .map((item) =>\n makeFinding({\n analyst_id: options.analystId,\n ...(options.producedAt === undefined ? {} : { produced_at: options.producedAt }),\n subject: item.check.subject,\n area: item.status === 'unresolved' ? 'assessment-coverage' : item.check.area,\n severity: item.status === 'unresolved' ? 'info' : item.check.severity,\n claim:\n item.status === 'supported'\n ? `Model-supported hypothesis: ${item.check.claim}`\n : `Unresolved review: ${item.check.claim}`,\n evidence_refs: structuredClone(item.check.evidence) as EvidenceRef[],\n // For unresolved findings, confidence describes the deterministic unresolved status only.\n confidence: item.status === 'supported' ? item.supportProbability : 1,\n derived_from_judge: true,\n id_basis: hashCanonical([item.question, item.status, review.version, item.check.claim]),\n metadata: {\n review_digest: report.reviewDigest,\n question: item.question,\n requested_model: report.requestedModel,\n served_model: report.servedModel,\n assessment: item.status,\n assessment_reason: item.reason,\n evidence_coverage: item.check.coverage,\n support_probability: item.supportProbability,\n refutation_probability: item.refutationProbability,\n unresolved_probability: item.unresolvedProbability,\n model_confidence: item.modelConfidence,\n calibration: 'not-established',\n confidence_basis:\n item.status === 'supported' ? 'model-support-mass' : 'deterministic-status',\n native_answer: structuredClone(result.answers[item.question]),\n },\n }),\n )\n}\n"],"mappings":";;;;;;AAYA,MAAM,OAAO,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC;AACpC,MAAM,cAAc,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;AACnD,MAAM,cAAc,EACjB,OAAO;CACN,OAAO;CACP,MAAM;CACN,SAAS;CACT,UAAU,EAAE,KAAK,kBAAkB;;CAEnC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;CACnC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;CAClC,gBAAgB;CAChB,eAAe;;CAEf,UAAU,EAAE,KAAK;EAAC;EAAY;EAAW;CAAS,CAAC;CACnD,UAAU,EAAE,MACV,EACG,OAAO;EACN,MAAM,EAAE,KAAK;GAAC;GAAQ;GAAS;GAAY;GAAW;EAAQ,CAAC;EAC/D,KAAK;EACL,SAAS,EAAE,OAAO,CAAC,CAAC,SAAS;CAC/B,CAAC,CAAC,CACD,OAAO,CACZ;AACF,CAAC,CAAC,CACD,OAAO,CAAC,CACR,QAAQ,UAAU,MAAM,iBAAiB,MAAM,gBAAgB,GAAG,EACjE,SAAS,qDACX,CAAC;AAyCH,SAAS,aAAa,UAA0C;CAC9D,IAAI,SAAS,SAAS,QAAQ,OAAO,CAAC,QAAQ,OAAO;CACrD,OAAO,SAAS,SAAS,WACrB,OAAO,KAAK,SAAS,QAAQ,IAC7B,SAAS,SAAS,KAAK,GAAG,UAAU,OAAO,KAAK,CAAC;AACvD;;AAGA,SAAgB,iBACd,OACsB;CACtB,IAAI,CAAC,MAAM,SAAS,KAAK,GAAG,MAAM,IAAI,UAAU,4BAA4B;CAC5E,gBAAgB,MAAM,OAAO;CAE7B,MAAM,WAAW,aAAa,KAAK;CACnC,gBAAgB,QAAQ;CAExB,MAAM,WAAW,gBAAgB,QAAQ;CACzC,MAAM,OAAO,OAAO,KAAK,SAAS,QAAQ,SAAS;CACnD,IACE,CAAC,SAAS,UACV,OAAO,KAAK,SAAS,MAAM,CAAC,CAAC,WAAW,KAAK,UAC7C,KAAK,MAAM,QAAQ,CAAC,OAAO,OAAO,SAAS,QAAQ,GAAG,CAAC,GAEvD,MAAM,IAAI,UAAU,mDAAmD;CAEzE,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,QAAQ,YAAY,MAAM,SAAS,OAAO,IAAI;EACpD,MAAM,SAAS,CAAC,GAAG,MAAM,UAAU,GAAG,MAAM,OAAO;EACnD,MAAM,UAAU,aAAa,SAAS,QAAQ,UAAU,IAAK;EAC7D,IACE,IAAI,IAAI,MAAM,CAAC,CAAC,SAAS,OAAO,UAChC,OAAO,MAAM,UAAU,CAAC,QAAQ,SAAS,KAAK,CAAC,GAE/C,MAAM,IAAI,UAAU,gDAAgD;EAEtE,IAAI,MAAM,aAAa,aAAa,MAAM,SAAS,WAAW,GAC5D,MAAM,IAAI,UAAU,kEAAkE;EAGxF,IAAI,gBAAgB,aAAa,KAAK,CAAC,MAAM,gBAAgB,SAAS,OAAO,IAAI,GAC/E,MAAM,IAAI,UAAU,gDAAgD;CAExE;CACA,MAAM,QAAQ,KAAK,KAAK,QAAQ,CAAC,KAAK,aAAa,SAAS,QAAQ,UAAU,IAAK,CAAC,CAAC;CACrF,OAAO,wBAAwB;EAAE,GAAG;EAAU,QAAQ,cAAc;GAAE;GAAU;EAAM,CAAC;CAAE,CAAC;AAC5F;;AAGA,SAAgB,gBACd,QACA,KACiB;CACjB,MAAM,EAAE,QAAQ,GAAG,UAAU;CAC7B,MAAM,WAAW,iBAAiB,KAAK;CACvC,IAAI,SAAS,WAAW,QAAQ,MAAM,IAAI,UAAU,6CAA6C;CACjG,MAAM,SAAS,eAAe,KAAK,SAAS,OAAO;CACnD,MAAM,cAAc,OAAO,QAAQ,SAAS,MAAM,CAAC,CAAC,KAAK,CAAC,UAAU,WAAW;EAC7E,MAAM,SAAS,OAAO,QAAQ;EAC9B,MAAM,gBACJ,OAAO,SAAS,SAAS;GAAE,MAAM,OAAO;GAAM,OAAO,IAAI,OAAO;EAAK,IAAI,OAAO;EAClF,MAAM,OAAO,WACX,OAAO,QAAQ,OAAO,UAAU,QAAQ,cAAc,QAAS,CAAC;EAClE,MAAM,qBAAqB,IAAI,MAAM,QAAQ;EAC7C,MAAM,wBAAwB,IAAI,MAAM,OAAO;EAC/C,MAAM,uBAAO,IAAI,IAAI,CAAC,GAAG,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC;EAC1D,MAAM,wBAAwB,IAC5B,OAAO,KAAK,aAAa,CAAC,CAAC,QAAQ,UAAU,CAAC,KAAK,IAAI,KAAK,CAAC,CAC/D;EACA,IAAI,SAAwC;EAC5C,IAAI,SAAwC;EAC5C,IAAI,qBAAqB,KAAK,wBAAwB,KAAK,wBAAwB,GACjF,MAAM,IAAI,UAAU,mEAAmE;EAEzF,IAAI,MAAM,aAAa,WAAW,SAAS;OACtC,IACH,sBAAsB,MAAM,kBAC5B,yBAAyB,MAAM,eAE/B,SAAS;OACJ,IAAI,sBAAsB,MAAM,gBAAgB;GACrD,SAAS;GACT,SAAS;EACX,OAAO,IAAI,yBAAyB,MAAM,eACxC,IAAI,MAAM,aAAa,YAAY;GACjC,SAAS;GACT,SAAS;EACX,OAAO,SAAS;EAElB,OAAO;GACL;GACA;GACA;GACA;GACA;GACA;GACA,iBAAiB,OAAO,SAAS,SAAS,OAAO,OAAO;GACxD,OAAO,gBAAgB,KAAK;EAC9B;CACF,CAAC;CACD,OAAO;EACL,cAAc;EACd,gBAAgB,OAAO,QAAQ;EAC/B,aAAa,OAAO;EACpB;CACF;AACF;;AAGA,SAAgB,kBACd,QACA,QACA,SACkB;CAClB,IAAI,CAAC,QAAQ,UAAU,KAAK,GAAG,MAAM,IAAI,UAAU,wBAAwB;CAC3E,MAAM,SAAS,gBAAgB,QAAQ,MAAM;CAC7C,OAAO,OAAO,YACX,QAAQ,SAAS,KAAK,WAAW,SAAS,CAAC,CAC3C,KAAK,SACJ,YAAY;EACV,YAAY,QAAQ;EACpB,GAAI,QAAQ,eAAe,KAAA,IAAY,CAAC,IAAI,EAAE,aAAa,QAAQ,WAAW;EAC9E,SAAS,KAAK,MAAM;EACpB,MAAM,KAAK,WAAW,eAAe,wBAAwB,KAAK,MAAM;EACxE,UAAU,KAAK,WAAW,eAAe,SAAS,KAAK,MAAM;EAC7D,OACE,KAAK,WAAW,cACZ,+BAA+B,KAAK,MAAM,UAC1C,sBAAsB,KAAK,MAAM;EACvC,eAAe,gBAAgB,KAAK,MAAM,QAAQ;EAElD,YAAY,KAAK,WAAW,cAAc,KAAK,qBAAqB;EACpE,oBAAoB;EACpB,UAAU,cAAc;GAAC,KAAK;GAAU,KAAK;GAAQ,OAAO;GAAS,KAAK,MAAM;EAAK,CAAC;EACtF,UAAU;GACR,eAAe,OAAO;GACtB,UAAU,KAAK;GACf,iBAAiB,OAAO;GACxB,cAAc,OAAO;GACrB,YAAY,KAAK;GACjB,mBAAmB,KAAK;GACxB,mBAAmB,KAAK,MAAM;GAC9B,qBAAqB,KAAK;GAC1B,wBAAwB,KAAK;GAC7B,wBAAwB,KAAK;GAC7B,kBAAkB,KAAK;GACvB,aAAa;GACb,kBACE,KAAK,WAAW,cAAc,uBAAuB;GACvD,eAAe,gBAAgB,OAAO,QAAQ,KAAK,SAAS;EAC9D;CACF,CAAC,CACH;AACJ"}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { i as AnalystFinding } from "./types-CeDfCRXp.js";
|
|
2
|
+
import { l as JevResult, o as JevQuestions, s as JevRequest } from "./jev-protocol-CpKI7XSu.js";
|
|
3
|
+
import { z } from "zod";
|
|
4
|
+
//#region src/jev-review.d.ts
|
|
5
|
+
declare const checkSchema: z.ZodObject<{
|
|
6
|
+
claim: z.ZodString;
|
|
7
|
+
area: z.ZodString;
|
|
8
|
+
subject: z.ZodString;
|
|
9
|
+
severity: z.ZodEnum<{
|
|
10
|
+
critical: "critical";
|
|
11
|
+
high: "high";
|
|
12
|
+
info: "info";
|
|
13
|
+
low: "low";
|
|
14
|
+
medium: "medium";
|
|
15
|
+
}>;
|
|
16
|
+
supports: z.ZodArray<z.ZodString>;
|
|
17
|
+
refutes: z.ZodArray<z.ZodString>;
|
|
18
|
+
supportAtLeast: z.ZodNumber;
|
|
19
|
+
refuteAtLeast: z.ZodNumber;
|
|
20
|
+
coverage: z.ZodEnum<{
|
|
21
|
+
complete: "complete";
|
|
22
|
+
missing: "missing";
|
|
23
|
+
partial: "partial";
|
|
24
|
+
}>;
|
|
25
|
+
evidence: z.ZodArray<z.ZodObject<{
|
|
26
|
+
kind: z.ZodEnum<{
|
|
27
|
+
artifact: "artifact";
|
|
28
|
+
event: "event";
|
|
29
|
+
finding: "finding";
|
|
30
|
+
metric: "metric";
|
|
31
|
+
span: "span";
|
|
32
|
+
}>;
|
|
33
|
+
uri: z.ZodString;
|
|
34
|
+
excerpt: z.ZodOptional<z.ZodString>;
|
|
35
|
+
}, z.core.$strict>>;
|
|
36
|
+
}, z.core.$strict>;
|
|
37
|
+
/** Optional interpretation of native choices/scores. Policy and evidence belong to the caller. */
|
|
38
|
+
type JevReviewCheck = z.infer<typeof checkSchema>;
|
|
39
|
+
interface JevReviewInput<Q extends JevQuestions = JevQuestions> {
|
|
40
|
+
version: string;
|
|
41
|
+
request: JevRequest<Q>;
|
|
42
|
+
checks: { [K in keyof Q]: JevReviewCheck; };
|
|
43
|
+
}
|
|
44
|
+
interface PreparedJevReview<Q extends JevQuestions = JevQuestions> extends JevReviewInput<Q> {
|
|
45
|
+
/** Binds questions, option order, context, evidence, thresholds and definition version. */
|
|
46
|
+
digest: string;
|
|
47
|
+
}
|
|
48
|
+
interface JevReviewAssessment {
|
|
49
|
+
question: string;
|
|
50
|
+
status: 'supported' | 'refuted' | 'unresolved';
|
|
51
|
+
reason: 'support-threshold' | 'refutation-threshold' | 'missing-evidence' | 'partial-negative-evidence' | 'below-threshold' | 'ambiguous-thresholds';
|
|
52
|
+
/** Unnormalized marginal mass of caller-designated native alternatives. Not calibrated risk. */
|
|
53
|
+
supportProbability: number;
|
|
54
|
+
refutationProbability: number;
|
|
55
|
+
unresolvedProbability: number;
|
|
56
|
+
modelConfidence: number | null;
|
|
57
|
+
check: JevReviewCheck;
|
|
58
|
+
}
|
|
59
|
+
interface JevReviewReport {
|
|
60
|
+
reviewDigest: string;
|
|
61
|
+
requestedModel: string;
|
|
62
|
+
servedModel: string;
|
|
63
|
+
assessments: JevReviewAssessment[];
|
|
64
|
+
}
|
|
65
|
+
/** Validate before paid admission; make a detached, frozen snapshot without rewriting native JSON. */
|
|
66
|
+
declare function prepareJevReview<const Q extends JevQuestions>(input: JevReviewInput<Q>): PreparedJevReview<Q>;
|
|
67
|
+
/** Re-score the same retained distribution without new inference or any authorization effect. */
|
|
68
|
+
declare function assessJevReview<Q extends JevQuestions>(review: PreparedJevReview<Q>, raw: unknown): JevReviewReport;
|
|
69
|
+
/** Ordinary analyst findings; a negative review is not a global safety certificate. */
|
|
70
|
+
declare function jevReviewFindings<Q extends JevQuestions>(review: PreparedJevReview<Q>, result: JevResult<Q>, options: {
|
|
71
|
+
analystId: string;
|
|
72
|
+
producedAt?: string;
|
|
73
|
+
}): AnalystFinding[];
|
|
74
|
+
//#endregion
|
|
75
|
+
export { PreparedJevReview as a, prepareJevReview as c, JevReviewReport as i, JevReviewCheck as n, assessJevReview as o, JevReviewInput as r, jevReviewFindings as s, JevReviewAssessment as t };
|
|
76
|
+
//# sourceMappingURL=jev-review-D4ynCORp.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"jev-review-D4ynCORp.d.ts","names":[],"sources":["../src/jev-review.ts"],"mappings":";;;;cAcM,aAAW,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0Bb,EAAA,KAAA;;KAGQ,iBAAiB,EAAE,aAAa;UAC3B,eAAe,UAAU,eAAe;EACvD;EACA,SAAS,WAAW;EACpB,WAAW,WAAW,IAAI;;UAGX,kBAAkB,UAAU,eAAe,sBAClD,eAAe;;EAEvB;;UAGe;EACf;EACA;EACA;;EAQA;EACA;EACA;EACA;EACA,OAAO;;UAGQ;EACf;EACA;EACA;EACA,aAAa;;;iBAWC,uBAAuB,UAAU,cAC/C,OAAO,eAAe,KACrB,kBAAkB;;iBAuCL,gBAAgB,UAAU,cACxC,QAAQ,kBAAkB,IAC1B,eACC;;iBAyDa,kBAAkB,UAAU,cAC1C,QAAQ,kBAAkB,IAC1B,QAAQ,UAAU,IAClB;EAAW;EAAmB;IAC7B"}
|
package/dist/jev.d.ts
CHANGED
|
@@ -1,80 +1,9 @@
|
|
|
1
1
|
import { b as CustomTokenPricing, g as CostReceiptInput } from "./cost-ledger-DbQdN3nO.js";
|
|
2
2
|
import { i as AnalystFinding, n as AnalystContext, t as Analyst } from "./types-CeDfCRXp.js";
|
|
3
|
-
import { R as Scenario, S as JudgeConfig, w as JudgeScore } from "./types-
|
|
3
|
+
import { R as Scenario, S as JudgeConfig, w as JudgeScore } from "./types-DfvQOe91.js";
|
|
4
4
|
import { EvaluationAnalystOptions, EvaluationContext, EvaluationJudgeOptions, EvaluationResult, Evaluator, EvaluatorOptions, asAnalyst, asJudge, createEvaluator } from "./evaluation.js";
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
//#region src/jev-review.d.ts
|
|
8
|
-
declare const checkSchema: z.ZodObject<{
|
|
9
|
-
claim: z.ZodString;
|
|
10
|
-
area: z.ZodString;
|
|
11
|
-
subject: z.ZodString;
|
|
12
|
-
severity: z.ZodEnum<{
|
|
13
|
-
critical: "critical";
|
|
14
|
-
high: "high";
|
|
15
|
-
info: "info";
|
|
16
|
-
low: "low";
|
|
17
|
-
medium: "medium";
|
|
18
|
-
}>;
|
|
19
|
-
supports: z.ZodArray<z.ZodString>;
|
|
20
|
-
refutes: z.ZodArray<z.ZodString>;
|
|
21
|
-
supportAtLeast: z.ZodNumber;
|
|
22
|
-
refuteAtLeast: z.ZodNumber;
|
|
23
|
-
coverage: z.ZodEnum<{
|
|
24
|
-
complete: "complete";
|
|
25
|
-
missing: "missing";
|
|
26
|
-
partial: "partial";
|
|
27
|
-
}>;
|
|
28
|
-
evidence: z.ZodArray<z.ZodObject<{
|
|
29
|
-
kind: z.ZodEnum<{
|
|
30
|
-
artifact: "artifact";
|
|
31
|
-
event: "event";
|
|
32
|
-
finding: "finding";
|
|
33
|
-
metric: "metric";
|
|
34
|
-
span: "span";
|
|
35
|
-
}>;
|
|
36
|
-
uri: z.ZodString;
|
|
37
|
-
excerpt: z.ZodOptional<z.ZodString>;
|
|
38
|
-
}, z.core.$strict>>;
|
|
39
|
-
}, z.core.$strict>;
|
|
40
|
-
/** Optional interpretation of native choices/scores. Policy and evidence belong to the caller. */
|
|
41
|
-
type JevReviewCheck = z.infer<typeof checkSchema>;
|
|
42
|
-
interface JevReviewInput<Q extends JevQuestions = JevQuestions> {
|
|
43
|
-
version: string;
|
|
44
|
-
request: JevRequest<Q>;
|
|
45
|
-
checks: { [K in keyof Q]: JevReviewCheck; };
|
|
46
|
-
}
|
|
47
|
-
interface PreparedJevReview<Q extends JevQuestions = JevQuestions> extends JevReviewInput<Q> {
|
|
48
|
-
/** Binds questions, option order, context, evidence, thresholds and definition version. */
|
|
49
|
-
digest: string;
|
|
50
|
-
}
|
|
51
|
-
interface JevReviewAssessment {
|
|
52
|
-
question: string;
|
|
53
|
-
status: 'supported' | 'refuted' | 'unresolved';
|
|
54
|
-
reason: 'support-threshold' | 'refutation-threshold' | 'missing-evidence' | 'partial-negative-evidence' | 'below-threshold' | 'ambiguous-thresholds';
|
|
55
|
-
/** Unnormalized marginal mass of caller-designated native alternatives. Not calibrated risk. */
|
|
56
|
-
supportProbability: number;
|
|
57
|
-
refutationProbability: number;
|
|
58
|
-
unresolvedProbability: number;
|
|
59
|
-
modelConfidence: number | null;
|
|
60
|
-
check: JevReviewCheck;
|
|
61
|
-
}
|
|
62
|
-
interface JevReviewReport {
|
|
63
|
-
reviewDigest: string;
|
|
64
|
-
requestedModel: string;
|
|
65
|
-
servedModel: string;
|
|
66
|
-
assessments: JevReviewAssessment[];
|
|
67
|
-
}
|
|
68
|
-
/** Validate before paid admission; make a detached, frozen snapshot without rewriting native JSON. */
|
|
69
|
-
declare function prepareJevReview<const Q extends JevQuestions>(input: JevReviewInput<Q>): PreparedJevReview<Q>;
|
|
70
|
-
/** Re-score the same retained distribution without new inference or any authorization effect. */
|
|
71
|
-
declare function assessJevReview<Q extends JevQuestions>(review: PreparedJevReview<Q>, raw: unknown): JevReviewReport;
|
|
72
|
-
/** Ordinary analyst findings; a negative review is not a global safety certificate. */
|
|
73
|
-
declare function jevReviewFindings<Q extends JevQuestions>(review: PreparedJevReview<Q>, result: JevResult<Q>, options: {
|
|
74
|
-
analystId: string;
|
|
75
|
-
producedAt?: string;
|
|
76
|
-
}): AnalystFinding[];
|
|
77
|
-
//#endregion
|
|
5
|
+
import { a as JevQuestion, c as JevResponseError, d as JevState, f as jevUsage, h as parseJevResult, i as JevJson, l as JevResult, m as parseJevRequest, n as JevAnswerFor, o as JevQuestions, p as parseJevQuestions, r as JevDescription, s as JevRequest, t as JevAnswer, u as JevScoreCriteria } from "./jev-protocol-CpKI7XSu.js";
|
|
6
|
+
import { a as PreparedJevReview, c as prepareJevReview, i as JevReviewReport, n as JevReviewCheck, o as assessJevReview, r as JevReviewInput, s as jevReviewFindings, t as JevReviewAssessment } from "./jev-review-D4ynCORp.js";
|
|
78
7
|
//#region src/jev.d.ts
|
|
79
8
|
type JevEvaluate = EvaluatorOptions<JevRequest, unknown>['execute'];
|
|
80
9
|
interface JevEvaluatorOptions {
|
package/dist/jev.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jev.d.ts","names":[],"sources":["../src/jev
|
|
1
|
+
{"version":3,"file":"jev.d.ts","names":[],"sources":["../src/jev.ts"],"mappings":";;;;;;;KAeY,cAAc,iBAAiB;UAC1B;EACf,UAAU;EACV,aAAa,iBAAiB;EAC9B,gBAAgB,iBAAiB;EACjC,UAAU;EACV,WAAW,sBAAsB;EACjC,oBAAoB,OAAO,UAAU;;EAErC,eAAe,mBAAmB;;;iBAIpB,aAAa,SAAS,6BA2BhB,UAAU,cAAY,SAC/B,WAAW,IAAE,UACZ,sBACT,QAAQ,iBAAiB,UAAU;KAUnC,MAAM,GAAG,UAAU;EAAc,UAAU;EAAG,UAAU;;KACxD,eAAe,GAAG,UAAU,gBAAgB,MAAM,OAAO,MAAM,IAAI,QAAQ;UAC/D,mBAAmB;EAClC;EACA;EACA,WAAW;;UAEI,gBACf,GACA,UAAU,WAAW,UACrB,UAAU,eAAe,sBACjB;EACR;EACA;EACA,WAAW,eAAe,MAAM,GAAG,IAAI;EACvC,cAAc,OAAO,MAAM,GAAG,IAAI,SAAS,sBAAsB,WAAW,QAAQ;EACpF,aAAa,YAAY,GAAG;EAC5B,UAAU;EACV,OAAO,OAAO,UAAU,IAAI,OAAO,MAAM,GAAG,OAAO,aAAa,QAAQ;EACxE,UAAU,QAAQ,iBAAiB,UAAU,KAAK,OAAO,MAAM,GAAG,cAAc;EAChF,aAAa,UAAU;;;iBA0BT,mBACd,QAAQ,WACR,UAAU,yBACT;;iBAgCa,SAAS,GAAG,UAAU,WAAW,UAAU,UAAU,eAAe,cAClF,cACA,SAAS,gBAAgB,GAAG,GAAG,KAC9B,YAAY,GAAG;UAqDD,kBAAkB,GAAG,UAAU,eAAe,sBACrD;EACR;EACA;EACA,WAAW,QAAQ;EACnB;EACA;EACA,WAAW,MAAM,OAAO,GAAG,SAAS,mBAAmB,IAAI,QAAQ;EACnE,cAAc,OAAO,GAAG,SAAS,mBAAmB,WAAW,QAAQ;EACvE,WACE,OAAO,UAAU,IACjB,OAAO,GACP,SAAS,mBACN,mBAAmB,QAAQ;EAChC,UACE,QAAQ,iBAAiB,UAAU,KACnC,OAAO,GACP,SAAS,0BACC;;iBAGE,WAAW,GAAG,UAAU,eAAe,cACrD,SAAS,kBAAkB,GAAG,KAC7B,QAAQ"}
|
package/dist/jev.js
CHANGED
|
@@ -1,158 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { o as jsonDocument } from "./canonical-BsZVgBhW.js";
|
|
2
2
|
import { s as weightedComposite } from "./descriptive-1V17A-qa.js";
|
|
3
3
|
import { n as contentHash } from "./verdict-cache-BQEsVy9n.js";
|
|
4
|
-
import { t as deepFreezeCanonicalJson } from "./deep-freeze-CurcOQg3.js";
|
|
5
|
-
import { r as makeFinding, t as ANALYST_SEVERITIES } from "./types-CW5b90JH.js";
|
|
6
4
|
import { asAnalyst, asJudge, createEvaluator } from "./evaluation.js";
|
|
7
5
|
import { JevResponseError, jevUsage, parseJevQuestions, parseJevRequest, parseJevResult } from "./jev-protocol.js";
|
|
8
|
-
import {
|
|
9
|
-
//#region src/jev-review.ts
|
|
10
|
-
const name = z.string().trim().min(1);
|
|
11
|
-
const probability = z.number().finite().gt(0).max(1);
|
|
12
|
-
const checkSchema = z.object({
|
|
13
|
-
claim: name,
|
|
14
|
-
area: name,
|
|
15
|
-
subject: name,
|
|
16
|
-
severity: z.enum(ANALYST_SEVERITIES),
|
|
17
|
-
/** Choice labels or score-level keys, never scores invented from category order. */
|
|
18
|
-
supports: z.array(z.string()).min(1),
|
|
19
|
-
refutes: z.array(z.string()).min(1),
|
|
20
|
-
supportAtLeast: probability,
|
|
21
|
-
refuteAtLeast: probability,
|
|
22
|
-
/** Host-supplied coverage of the question's required evidence, not model self-assessment. */
|
|
23
|
-
coverage: z.enum([
|
|
24
|
-
"complete",
|
|
25
|
-
"partial",
|
|
26
|
-
"missing"
|
|
27
|
-
]),
|
|
28
|
-
evidence: z.array(z.object({
|
|
29
|
-
kind: z.enum([
|
|
30
|
-
"span",
|
|
31
|
-
"event",
|
|
32
|
-
"artifact",
|
|
33
|
-
"finding",
|
|
34
|
-
"metric"
|
|
35
|
-
]),
|
|
36
|
-
uri: name,
|
|
37
|
-
excerpt: z.string().optional()
|
|
38
|
-
}).strict())
|
|
39
|
-
}).strict().refine((value) => value.supportAtLeast + value.refuteAtLeast > 1, { message: "Support and refutation thresholds must not overlap" });
|
|
40
|
-
function alternatives(question) {
|
|
41
|
-
if (question.type === "noul") return ["true", "false"];
|
|
42
|
-
return question.type === "choice" ? Object.keys(question.criteria) : question.criteria.map((_, index) => String(index));
|
|
43
|
-
}
|
|
44
|
-
/** Validate before paid admission; make a detached, frozen snapshot without rewriting native JSON. */
|
|
45
|
-
function prepareJevReview(input) {
|
|
46
|
-
if (!input.version?.trim()) throw new TypeError("Review version is required");
|
|
47
|
-
parseJevRequest(input.request);
|
|
48
|
-
const document = jsonDocument(input);
|
|
49
|
-
canonicalString(document);
|
|
50
|
-
const snapshot = structuredClone(document);
|
|
51
|
-
const keys = Object.keys(snapshot.request.questions);
|
|
52
|
-
if (!snapshot.checks || Object.keys(snapshot.checks).length !== keys.length || keys.some((key) => !Object.hasOwn(snapshot.checks, key))) throw new TypeError("Review checks must match the exact question names");
|
|
53
|
-
for (const key of keys) {
|
|
54
|
-
const check = checkSchema.parse(snapshot.checks[key]);
|
|
55
|
-
const labels = [...check.supports, ...check.refutes];
|
|
56
|
-
const allowed = alternatives(snapshot.request.questions[key]);
|
|
57
|
-
if (new Set(labels).size !== labels.length || labels.some((label) => !allowed.includes(label))) throw new TypeError("Review alternatives must exist and be disjoint");
|
|
58
|
-
if (check.coverage !== "missing" && check.evidence.length === 0) throw new TypeError("Review evidence references are required for non-missing coverage");
|
|
59
|
-
if (canonicalString(jsonDocument(check)) !== canonicalString(snapshot.checks[key])) throw new TypeError("Review metadata must not require normalization");
|
|
60
|
-
}
|
|
61
|
-
const order = keys.map((key) => [key, alternatives(snapshot.request.questions[key])]);
|
|
62
|
-
return deepFreezeCanonicalJson({
|
|
63
|
-
...snapshot,
|
|
64
|
-
digest: hashCanonical({
|
|
65
|
-
snapshot,
|
|
66
|
-
order
|
|
67
|
-
})
|
|
68
|
-
});
|
|
69
|
-
}
|
|
70
|
-
/** Re-score the same retained distribution without new inference or any authorization effect. */
|
|
71
|
-
function assessJevReview(review, raw) {
|
|
72
|
-
const { digest, ...input } = review;
|
|
73
|
-
const prepared = prepareJevReview(input);
|
|
74
|
-
if (prepared.digest !== digest) throw new TypeError("Review definition changed after preparation");
|
|
75
|
-
const result = parseJevResult(raw, prepared.request);
|
|
76
|
-
const assessments = Object.entries(prepared.checks).map(([question, check]) => {
|
|
77
|
-
const answer = result.answers[question];
|
|
78
|
-
const probabilities = answer.type === "noul" ? {
|
|
79
|
-
true: answer.noul,
|
|
80
|
-
false: 1 - answer.noul
|
|
81
|
-
} : answer.probabilities;
|
|
82
|
-
const sum = (labels) => labels.reduce((total, label) => total + probabilities[label], 0);
|
|
83
|
-
const supportProbability = sum(check.supports);
|
|
84
|
-
const refutationProbability = sum(check.refutes);
|
|
85
|
-
const used = /* @__PURE__ */ new Set([...check.supports, ...check.refutes]);
|
|
86
|
-
const unresolvedProbability = sum(Object.keys(probabilities).filter((label) => !used.has(label)));
|
|
87
|
-
let status = "unresolved";
|
|
88
|
-
let reason = "below-threshold";
|
|
89
|
-
if (supportProbability > 1 || refutationProbability > 1 || unresolvedProbability > 1) throw new TypeError("Review probability mass exceeds one; do not silently normalize it");
|
|
90
|
-
if (check.coverage === "missing") reason = "missing-evidence";
|
|
91
|
-
else if (supportProbability >= check.supportAtLeast && refutationProbability >= check.refuteAtLeast) reason = "ambiguous-thresholds";
|
|
92
|
-
else if (supportProbability >= check.supportAtLeast) {
|
|
93
|
-
status = "supported";
|
|
94
|
-
reason = "support-threshold";
|
|
95
|
-
} else if (refutationProbability >= check.refuteAtLeast) if (check.coverage === "complete") {
|
|
96
|
-
status = "refuted";
|
|
97
|
-
reason = "refutation-threshold";
|
|
98
|
-
} else reason = "partial-negative-evidence";
|
|
99
|
-
return {
|
|
100
|
-
question,
|
|
101
|
-
status,
|
|
102
|
-
reason,
|
|
103
|
-
supportProbability,
|
|
104
|
-
refutationProbability,
|
|
105
|
-
unresolvedProbability,
|
|
106
|
-
modelConfidence: answer.type === "noul" ? null : answer.confidence,
|
|
107
|
-
check: structuredClone(check)
|
|
108
|
-
};
|
|
109
|
-
});
|
|
110
|
-
return {
|
|
111
|
-
reviewDigest: digest,
|
|
112
|
-
requestedModel: review.request.model,
|
|
113
|
-
servedModel: result.model,
|
|
114
|
-
assessments
|
|
115
|
-
};
|
|
116
|
-
}
|
|
117
|
-
/** Ordinary analyst findings; a negative review is not a global safety certificate. */
|
|
118
|
-
function jevReviewFindings(review, result, options) {
|
|
119
|
-
if (!options.analystId.trim()) throw new TypeError("Analyst id is required");
|
|
120
|
-
const report = assessJevReview(review, result);
|
|
121
|
-
return report.assessments.filter((item) => item.status !== "refuted").map((item) => makeFinding({
|
|
122
|
-
analyst_id: options.analystId,
|
|
123
|
-
...options.producedAt === void 0 ? {} : { produced_at: options.producedAt },
|
|
124
|
-
subject: item.check.subject,
|
|
125
|
-
area: item.status === "unresolved" ? "assessment-coverage" : item.check.area,
|
|
126
|
-
severity: item.status === "unresolved" ? "info" : item.check.severity,
|
|
127
|
-
claim: item.status === "supported" ? `Model-supported hypothesis: ${item.check.claim}` : `Unresolved review: ${item.check.claim}`,
|
|
128
|
-
evidence_refs: structuredClone(item.check.evidence),
|
|
129
|
-
confidence: item.status === "supported" ? item.supportProbability : 1,
|
|
130
|
-
derived_from_judge: true,
|
|
131
|
-
id_basis: hashCanonical([
|
|
132
|
-
item.question,
|
|
133
|
-
item.status,
|
|
134
|
-
review.version,
|
|
135
|
-
item.check.claim
|
|
136
|
-
]),
|
|
137
|
-
metadata: {
|
|
138
|
-
review_digest: report.reviewDigest,
|
|
139
|
-
question: item.question,
|
|
140
|
-
requested_model: report.requestedModel,
|
|
141
|
-
served_model: report.servedModel,
|
|
142
|
-
assessment: item.status,
|
|
143
|
-
assessment_reason: item.reason,
|
|
144
|
-
evidence_coverage: item.check.coverage,
|
|
145
|
-
support_probability: item.supportProbability,
|
|
146
|
-
refutation_probability: item.refutationProbability,
|
|
147
|
-
unresolved_probability: item.unresolvedProbability,
|
|
148
|
-
model_confidence: item.modelConfidence,
|
|
149
|
-
calibration: "not-established",
|
|
150
|
-
confidence_basis: item.status === "supported" ? "model-support-mass" : "deterministic-status",
|
|
151
|
-
native_answer: structuredClone(result.answers[item.question])
|
|
152
|
-
}
|
|
153
|
-
}));
|
|
154
|
-
}
|
|
155
|
-
//#endregion
|
|
6
|
+
import { n as jevReviewFindings, r as prepareJevReview, t as assessJevReview } from "./jev-review-Cu9xtIu4.js";
|
|
156
7
|
//#region src/jev.ts
|
|
157
8
|
/** Configure transport once; supply native state, questions, and model on every invocation. */
|
|
158
9
|
function jevEvaluator(options) {
|
package/dist/jev.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jev.js","names":[],"sources":["../src/jev-review.ts","../src/jev.ts"],"sourcesContent":["import { z } from 'zod'\nimport {\n ANALYST_SEVERITIES,\n type AnalystFinding,\n type EvidenceRef,\n makeFinding,\n} from './analyst/types'\nimport type { JevQuestions, JevRequest, JevResult } from './jev-protocol'\nimport { parseJevRequest, parseJevResult } from './jev-protocol'\nimport { canonicalString, hashCanonical, jsonDocument } from './ledger-core/canonical'\nimport { deepFreezeCanonicalJson } from './ledger-core/deep-freeze'\n\nconst name = z.string().trim().min(1)\nconst probability = z.number().finite().gt(0).max(1)\nconst checkSchema = z\n .object({\n claim: name,\n area: name,\n subject: name,\n severity: z.enum(ANALYST_SEVERITIES),\n /** Choice labels or score-level keys, never scores invented from category order. */\n supports: z.array(z.string()).min(1),\n refutes: z.array(z.string()).min(1),\n supportAtLeast: probability,\n refuteAtLeast: probability,\n /** Host-supplied coverage of the question's required evidence, not model self-assessment. */\n coverage: z.enum(['complete', 'partial', 'missing']),\n evidence: z.array(\n z\n .object({\n kind: z.enum(['span', 'event', 'artifact', 'finding', 'metric']),\n uri: name,\n excerpt: z.string().optional(),\n })\n .strict(),\n ),\n })\n .strict()\n .refine((value) => value.supportAtLeast + value.refuteAtLeast > 1, {\n message: 'Support and refutation thresholds must not overlap',\n })\n\n/** Optional interpretation of native choices/scores. Policy and evidence belong to the caller. */\nexport type JevReviewCheck = z.infer<typeof checkSchema>\nexport interface JevReviewInput<Q extends JevQuestions = JevQuestions> {\n version: string\n request: JevRequest<Q>\n checks: { [K in keyof Q]: JevReviewCheck }\n}\n\nexport interface PreparedJevReview<Q extends JevQuestions = JevQuestions>\n extends JevReviewInput<Q> {\n /** Binds questions, option order, context, evidence, thresholds and definition version. */\n digest: string\n}\n\nexport interface JevReviewAssessment {\n question: string\n status: 'supported' | 'refuted' | 'unresolved'\n reason:\n | 'support-threshold'\n | 'refutation-threshold'\n | 'missing-evidence'\n | 'partial-negative-evidence'\n | 'below-threshold'\n | 'ambiguous-thresholds'\n /** Unnormalized marginal mass of caller-designated native alternatives. Not calibrated risk. */\n supportProbability: number\n refutationProbability: number\n unresolvedProbability: number\n modelConfidence: number | null\n check: JevReviewCheck\n}\n\nexport interface JevReviewReport {\n reviewDigest: string\n requestedModel: string\n servedModel: string\n assessments: JevReviewAssessment[]\n}\n\nfunction alternatives(question: JevQuestions[string]): string[] {\n if (question.type === 'noul') return ['true', 'false']\n return question.type === 'choice'\n ? Object.keys(question.criteria)\n : question.criteria.map((_, index) => String(index))\n}\n\n/** Validate before paid admission; make a detached, frozen snapshot without rewriting native JSON. */\nexport function prepareJevReview<const Q extends JevQuestions>(\n input: JevReviewInput<Q>,\n): PreparedJevReview<Q> {\n if (!input.version?.trim()) throw new TypeError('Review version is required')\n parseJevRequest(input.request)\n // Document form drops only optional undefined object fields; canonicalization rejects loss.\n const document = jsonDocument(input)\n canonicalString(document)\n // Canonical bytes are for hashing, not dispatch: option order can affect a classifier.\n const snapshot = structuredClone(document) as JevReviewInput<Q>\n const keys = Object.keys(snapshot.request.questions)\n if (\n !snapshot.checks ||\n Object.keys(snapshot.checks).length !== keys.length ||\n keys.some((key) => !Object.hasOwn(snapshot.checks, key))\n ) {\n throw new TypeError('Review checks must match the exact question names')\n }\n for (const key of keys) {\n const check = checkSchema.parse(snapshot.checks[key])\n const labels = [...check.supports, ...check.refutes]\n const allowed = alternatives(snapshot.request.questions[key]!)\n if (\n new Set(labels).size !== labels.length ||\n labels.some((label) => !allowed.includes(label))\n ) {\n throw new TypeError('Review alternatives must exist and be disjoint')\n }\n if (check.coverage !== 'missing' && check.evidence.length === 0) {\n throw new TypeError('Review evidence references are required for non-missing coverage')\n }\n // Validation must not normalize a label into a different native option.\n if (canonicalString(jsonDocument(check)) !== canonicalString(snapshot.checks[key])) {\n throw new TypeError('Review metadata must not require normalization')\n }\n }\n const order = keys.map((key) => [key, alternatives(snapshot.request.questions[key]!)])\n return deepFreezeCanonicalJson({ ...snapshot, digest: hashCanonical({ snapshot, order }) })\n}\n\n/** Re-score the same retained distribution without new inference or any authorization effect. */\nexport function assessJevReview<Q extends JevQuestions>(\n review: PreparedJevReview<Q>,\n raw: unknown,\n): JevReviewReport {\n const { digest, ...input } = review\n const prepared = prepareJevReview(input)\n if (prepared.digest !== digest) throw new TypeError('Review definition changed after preparation')\n const result = parseJevResult(raw, prepared.request)\n const assessments = Object.entries(prepared.checks).map(([question, check]) => {\n const answer = result.answers[question]!\n const probabilities: Record<string, number> =\n answer.type === 'noul' ? { true: answer.noul, false: 1 - answer.noul } : answer.probabilities\n const sum = (labels: string[]) =>\n labels.reduce((total, label) => total + probabilities[label]!, 0)\n const supportProbability = sum(check.supports)\n const refutationProbability = sum(check.refutes)\n const used = new Set([...check.supports, ...check.refutes])\n const unresolvedProbability = sum(\n Object.keys(probabilities).filter((label) => !used.has(label)),\n )\n let status: JevReviewAssessment['status'] = 'unresolved'\n let reason: JevReviewAssessment['reason'] = 'below-threshold'\n if (supportProbability > 1 || refutationProbability > 1 || unresolvedProbability > 1) {\n throw new TypeError('Review probability mass exceeds one; do not silently normalize it')\n }\n if (check.coverage === 'missing') reason = 'missing-evidence'\n else if (\n supportProbability >= check.supportAtLeast &&\n refutationProbability >= check.refuteAtLeast\n ) {\n reason = 'ambiguous-thresholds'\n } else if (supportProbability >= check.supportAtLeast) {\n status = 'supported'\n reason = 'support-threshold'\n } else if (refutationProbability >= check.refuteAtLeast) {\n if (check.coverage === 'complete') {\n status = 'refuted'\n reason = 'refutation-threshold'\n } else reason = 'partial-negative-evidence'\n }\n return {\n question,\n status,\n reason,\n supportProbability,\n refutationProbability,\n unresolvedProbability,\n modelConfidence: answer.type === 'noul' ? null : answer.confidence,\n check: structuredClone(check),\n }\n })\n return {\n reviewDigest: digest,\n requestedModel: review.request.model,\n servedModel: result.model,\n assessments,\n }\n}\n\n/** Ordinary analyst findings; a negative review is not a global safety certificate. */\nexport function jevReviewFindings<Q extends JevQuestions>(\n review: PreparedJevReview<Q>,\n result: JevResult<Q>,\n options: { analystId: string; producedAt?: string },\n): AnalystFinding[] {\n if (!options.analystId.trim()) throw new TypeError('Analyst id is required')\n const report = assessJevReview(review, result)\n return report.assessments\n .filter((item) => item.status !== 'refuted')\n .map((item) =>\n makeFinding({\n analyst_id: options.analystId,\n ...(options.producedAt === undefined ? {} : { produced_at: options.producedAt }),\n subject: item.check.subject,\n area: item.status === 'unresolved' ? 'assessment-coverage' : item.check.area,\n severity: item.status === 'unresolved' ? 'info' : item.check.severity,\n claim:\n item.status === 'supported'\n ? `Model-supported hypothesis: ${item.check.claim}`\n : `Unresolved review: ${item.check.claim}`,\n evidence_refs: structuredClone(item.check.evidence) as EvidenceRef[],\n // For unresolved findings, confidence describes the deterministic unresolved status only.\n confidence: item.status === 'supported' ? item.supportProbability : 1,\n derived_from_judge: true,\n id_basis: hashCanonical([item.question, item.status, review.version, item.check.claim]),\n metadata: {\n review_digest: report.reviewDigest,\n question: item.question,\n requested_model: report.requestedModel,\n served_model: report.servedModel,\n assessment: item.status,\n assessment_reason: item.reason,\n evidence_coverage: item.check.coverage,\n support_probability: item.supportProbability,\n refutation_probability: item.refutationProbability,\n unresolved_probability: item.unresolvedProbability,\n model_confidence: item.modelConfidence,\n calibration: 'not-established',\n confidence_basis:\n item.status === 'supported' ? 'model-support-mass' : 'deterministic-status',\n native_answer: structuredClone(result.answers[item.question]),\n },\n }),\n )\n}\n","import type { Analyst, AnalystContext, AnalystFinding } from './analyst/types'\nimport type { JudgeConfig, JudgeScore, Scenario } from './campaign/types'\nimport type { CostReceiptInput, CustomTokenPricing } from './cost-ledger'\nimport type { EvaluationContext, EvaluationResult, EvaluatorOptions } from './evaluation'\nimport { asAnalyst, asJudge, createEvaluator } from './evaluation'\nimport type { JevQuestions, JevRequest, JevResult, JevState } from './jev-protocol'\nimport { jevUsage, parseJevQuestions, parseJevRequest, parseJevResult } from './jev-protocol'\nimport { jsonDocument } from './ledger-core/canonical'\nimport { weightedComposite } from './statistics'\nimport { contentHash } from './verdict-cache'\n\nexport * from './evaluation'\nexport * from './jev-protocol'\nexport * from './jev-review'\n\nexport type JevEvaluate = EvaluatorOptions<JevRequest, unknown>['execute']\nexport interface JevEvaluatorOptions {\n evaluate: JevEvaluate\n costLedger?: EvaluatorOptions<JevRequest, unknown>['costLedger']\n maximumCharge?: EvaluatorOptions<JevRequest, unknown>['maximumCharge']\n pricing?: CustomTokenPricing\n receipt?: (response: unknown) => CostReceiptInput\n receiptFromError?: (error: Error) => CostReceiptInput | undefined\n /** Optional deployment policy; the protocol adapter does not hardcode model aliases. */\n acceptModel?: (requested: string, served: string) => boolean\n}\n\n/** Configure transport once; supply native state, questions, and model on every invocation. */\nexport function jevEvaluator(options: JevEvaluatorOptions) {\n const config = { ...options, pricing: options.pricing && { ...options.pricing } }\n const run = createEvaluator<JevRequest, unknown>({\n execute: (request, context) => config.evaluate(structuredClone(request), context),\n model: (request) => request.model,\n costLedger: config.costLedger,\n maximumCharge: config.maximumCharge,\n receiptFromError: config.receiptFromError,\n receipt:\n config.receipt ??\n ((raw) => {\n const usage = jevUsage(raw)\n const { model } = raw as JevResult\n return {\n model,\n inputTokens: usage.input_tokens,\n outputTokens: usage.output_tokens,\n customTokenPricing: config.pricing,\n }\n }),\n validate: (raw, request) => {\n const result = parseJevResult(raw, request)\n if (config.acceptModel && !config.acceptModel(request.model, result.model)) {\n throw new Error('Evaluation served a model rejected by the caller policy')\n }\n },\n })\n return async <const Q extends JevQuestions>(\n request: JevRequest<Q>,\n context?: EvaluationContext,\n ): Promise<EvaluationResult<JevResult<Q>>> => {\n context?.signal?.throwIfAborted()\n parseJevRequest(request)\n const snapshot = structuredClone(request)\n const result = await run(snapshot, context)\n // The validator checked these exact questions after recording the paid work.\n return { ...result, value: result.value as JevResult<Q> }\n }\n}\n\ntype Input<A, S extends Scenario> = { artifact: A; scenario: S }\ntype QuestionSource<I, Q extends JevQuestions> = Q | ((input: I) => Q | Promise<Q>)\nexport interface JevOptions extends JevEvaluatorOptions {\n model: string\n version: string\n questions: JevQuestions\n}\nexport interface JevJudgeOptions<\n A,\n S extends Scenario = Scenario,\n Q extends JevQuestions = JevQuestions,\n> extends JevEvaluatorOptions {\n model: string\n version: string\n questions: QuestionSource<Input<A, S>, Q>\n renderState: (input: Input<A, S>, context: EvaluationContext) => JevState | Promise<JevState>\n dimensions?: JudgeConfig<A, S>['dimensions']\n weights?: Record<string, number>\n map?: (value: JevResult<Q>, input: Input<A, S>) => JudgeScore | Promise<JudgeScore>\n record?: (result: EvaluationResult<JevResult<Q>>, input: Input<A, S>) => void | Promise<void>\n appliesTo?: (scenario: S) => boolean\n}\n\n/** Canonical identity also retains native question/alternative order; dispatch stays untouched. */\nfunction questionIdentity(questions: JevQuestions) {\n return {\n document: jsonDocument(questions),\n order: Object.entries(questions).map(([name, question]) => [\n name,\n question.type === 'choice' ? Object.keys(question.criteria) : null,\n ]),\n }\n}\n\nfunction validateWeights(weights: Record<string, number>, keys: string[]): void {\n if (\n Object.entries(weights).some(\n ([key, weight]) => !keys.includes(key) || !Number.isFinite(weight) || weight < 0,\n ) ||\n Object.values(weights).reduce((sum, weight) => sum + weight, 0) <= 0\n ) {\n throw new TypeError('Invalid judge weights')\n }\n}\n\n/** Optional convention. Choice utilities and non-uniform scales belong in an explicit map. */\nexport function normalizedJevScore(\n result: JevResult,\n weights?: Record<string, number>,\n): JudgeScore {\n const dimensions: Record<string, number> = Object.create(null)\n const distribution: NonNullable<JudgeScore['distribution']> = Object.create(null)\n for (const [key, answer] of Object.entries(result.answers)) {\n if (answer.type === 'choice') throw new TypeError('Choice scoring requires an explicit map')\n if (answer.type === 'noul') {\n dimensions[key] = answer.noul\n distribution[key] = [\n { score: 0, probability: 1 - answer.noul },\n { score: 1, probability: answer.noul },\n ]\n } else {\n const scale = Object.keys(answer.legend).length - 1\n dimensions[key] = answer.score / scale\n distribution[key] = Object.entries(answer.probabilities).map(([level, probability]) => ({\n score: Number(level) / scale,\n probability,\n }))\n }\n }\n const effective = weights ?? Object.fromEntries(Object.keys(dimensions).map((key) => [key, 1]))\n validateWeights(effective, Object.keys(dimensions))\n return {\n dimensions,\n composite: weightedComposite({ dims: dimensions, weights: effective }).composite,\n notes: '',\n scoringMethod: 'expectation',\n distribution,\n }\n}\n\n/** Convenience around the public evaluator. No domain questions, thresholds, or prose. */\nexport function jevJudge<A, S extends Scenario = Scenario, Q extends JevQuestions = JevQuestions>(\n name: string,\n options: JevJudgeOptions<A, S, Q>,\n): JudgeConfig<A, S> {\n const config = { ...options }\n const source =\n typeof options.questions === 'function' ? options.questions : structuredClone(options.questions)\n const weights = options.weights && { ...options.weights }\n if (!name.trim() || !options.version.trim()) {\n throw new TypeError('Judge name and version are required')\n }\n if ((typeof source === 'function' || config.map) && !config.dimensions) {\n throw new TypeError('Dynamic questions and custom mappings require stable judge dimensions')\n }\n if (!config.map && typeof source !== 'function') {\n parseJevQuestions(source)\n if (Object.values(source).some((question) => question.type === 'choice')) {\n throw new TypeError(\n 'Choice scoring requires an explicit map; labels have no numeric ordering',\n )\n }\n if (weights) validateWeights(weights, Object.keys(source))\n }\n const dimensions =\n config.dimensions ??\n Object.entries(source).map(([key, question]) => ({\n key,\n description: typeof question.instructions === 'string' ? question.instructions : key,\n }))\n const evaluate = jevEvaluator(config)\n return asJudge<A, S, JevResult<Q>>({\n name,\n version: contentHash({\n model: config.model,\n version: config.version,\n questions: typeof source === 'function' ? 'dynamic' : questionIdentity(source),\n dimensions,\n // Hash the JSON document: optional undefined fields are absent on the wire.\n ...(weights ? { weights } : {}),\n }),\n dimensions,\n appliesTo: config.appliesTo,\n evaluate: async (input, context = {}) =>\n evaluate(\n {\n model: config.model,\n state: await config.renderState(input, context),\n questions: typeof source === 'function' ? await source(input) : source,\n },\n context,\n ),\n map: config.map ?? ((value) => normalizedJevScore(value, weights)),\n record: config.record,\n })\n}\n\nexport interface JevAnalystOptions<I, Q extends JevQuestions = JevQuestions>\n extends JevEvaluatorOptions {\n id: string\n description: string\n inputKind: Analyst<I>['inputKind']\n model: string\n version: string\n questions: Q | ((input: I, context: AnalystContext) => Q | Promise<Q>)\n renderState: (input: I, context: AnalystContext) => JevState | Promise<JevState>\n findings: (\n value: JevResult<Q>,\n input: I,\n context: AnalystContext,\n ) => AnalystFinding[] | Promise<AnalystFinding[]>\n record?: (\n result: EvaluationResult<JevResult<Q>>,\n input: I,\n context: AnalystContext,\n ) => void | Promise<void>\n}\n\nexport function jevAnalyst<I, Q extends JevQuestions = JevQuestions>(\n options: JevAnalystOptions<I, Q>,\n): Analyst<I> {\n const config = { ...options }\n const source =\n typeof options.questions === 'function' ? options.questions : structuredClone(options.questions)\n if (!config.id.trim() || !config.version.trim() || !config.description.trim()) {\n throw new TypeError('Analyst id, version, and description are required')\n }\n const evaluate = jevEvaluator(config)\n return asAnalyst<I, JevResult<Q>>({\n id: config.id,\n description: config.description,\n inputKind: config.inputKind,\n version: contentHash({\n model: config.model,\n version: config.version,\n questions: typeof source === 'function' ? 'dynamic' : questionIdentity(source),\n }),\n cost: { kind: 'llm', models: [config.model] },\n evaluate: async (input, context, analystContext) =>\n evaluate(\n {\n model: config.model,\n state: await config.renderState(input, analystContext),\n questions: typeof source === 'function' ? await source(input, analystContext) : source,\n },\n context,\n ),\n map: config.findings,\n record: config.record,\n })\n}\n"],"mappings":";;;;;;;;;AAYA,MAAM,OAAO,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC;AACpC,MAAM,cAAc,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;AACnD,MAAM,cAAc,EACjB,OAAO;CACN,OAAO;CACP,MAAM;CACN,SAAS;CACT,UAAU,EAAE,KAAK,kBAAkB;;CAEnC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;CACnC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;CAClC,gBAAgB;CAChB,eAAe;;CAEf,UAAU,EAAE,KAAK;EAAC;EAAY;EAAW;CAAS,CAAC;CACnD,UAAU,EAAE,MACV,EACG,OAAO;EACN,MAAM,EAAE,KAAK;GAAC;GAAQ;GAAS;GAAY;GAAW;EAAQ,CAAC;EAC/D,KAAK;EACL,SAAS,EAAE,OAAO,CAAC,CAAC,SAAS;CAC/B,CAAC,CAAC,CACD,OAAO,CACZ;AACF,CAAC,CAAC,CACD,OAAO,CAAC,CACR,QAAQ,UAAU,MAAM,iBAAiB,MAAM,gBAAgB,GAAG,EACjE,SAAS,qDACX,CAAC;AAyCH,SAAS,aAAa,UAA0C;CAC9D,IAAI,SAAS,SAAS,QAAQ,OAAO,CAAC,QAAQ,OAAO;CACrD,OAAO,SAAS,SAAS,WACrB,OAAO,KAAK,SAAS,QAAQ,IAC7B,SAAS,SAAS,KAAK,GAAG,UAAU,OAAO,KAAK,CAAC;AACvD;;AAGA,SAAgB,iBACd,OACsB;CACtB,IAAI,CAAC,MAAM,SAAS,KAAK,GAAG,MAAM,IAAI,UAAU,4BAA4B;CAC5E,gBAAgB,MAAM,OAAO;CAE7B,MAAM,WAAW,aAAa,KAAK;CACnC,gBAAgB,QAAQ;CAExB,MAAM,WAAW,gBAAgB,QAAQ;CACzC,MAAM,OAAO,OAAO,KAAK,SAAS,QAAQ,SAAS;CACnD,IACE,CAAC,SAAS,UACV,OAAO,KAAK,SAAS,MAAM,CAAC,CAAC,WAAW,KAAK,UAC7C,KAAK,MAAM,QAAQ,CAAC,OAAO,OAAO,SAAS,QAAQ,GAAG,CAAC,GAEvD,MAAM,IAAI,UAAU,mDAAmD;CAEzE,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,QAAQ,YAAY,MAAM,SAAS,OAAO,IAAI;EACpD,MAAM,SAAS,CAAC,GAAG,MAAM,UAAU,GAAG,MAAM,OAAO;EACnD,MAAM,UAAU,aAAa,SAAS,QAAQ,UAAU,IAAK;EAC7D,IACE,IAAI,IAAI,MAAM,CAAC,CAAC,SAAS,OAAO,UAChC,OAAO,MAAM,UAAU,CAAC,QAAQ,SAAS,KAAK,CAAC,GAE/C,MAAM,IAAI,UAAU,gDAAgD;EAEtE,IAAI,MAAM,aAAa,aAAa,MAAM,SAAS,WAAW,GAC5D,MAAM,IAAI,UAAU,kEAAkE;EAGxF,IAAI,gBAAgB,aAAa,KAAK,CAAC,MAAM,gBAAgB,SAAS,OAAO,IAAI,GAC/E,MAAM,IAAI,UAAU,gDAAgD;CAExE;CACA,MAAM,QAAQ,KAAK,KAAK,QAAQ,CAAC,KAAK,aAAa,SAAS,QAAQ,UAAU,IAAK,CAAC,CAAC;CACrF,OAAO,wBAAwB;EAAE,GAAG;EAAU,QAAQ,cAAc;GAAE;GAAU;EAAM,CAAC;CAAE,CAAC;AAC5F;;AAGA,SAAgB,gBACd,QACA,KACiB;CACjB,MAAM,EAAE,QAAQ,GAAG,UAAU;CAC7B,MAAM,WAAW,iBAAiB,KAAK;CACvC,IAAI,SAAS,WAAW,QAAQ,MAAM,IAAI,UAAU,6CAA6C;CACjG,MAAM,SAAS,eAAe,KAAK,SAAS,OAAO;CACnD,MAAM,cAAc,OAAO,QAAQ,SAAS,MAAM,CAAC,CAAC,KAAK,CAAC,UAAU,WAAW;EAC7E,MAAM,SAAS,OAAO,QAAQ;EAC9B,MAAM,gBACJ,OAAO,SAAS,SAAS;GAAE,MAAM,OAAO;GAAM,OAAO,IAAI,OAAO;EAAK,IAAI,OAAO;EAClF,MAAM,OAAO,WACX,OAAO,QAAQ,OAAO,UAAU,QAAQ,cAAc,QAAS,CAAC;EAClE,MAAM,qBAAqB,IAAI,MAAM,QAAQ;EAC7C,MAAM,wBAAwB,IAAI,MAAM,OAAO;EAC/C,MAAM,uBAAO,IAAI,IAAI,CAAC,GAAG,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC;EAC1D,MAAM,wBAAwB,IAC5B,OAAO,KAAK,aAAa,CAAC,CAAC,QAAQ,UAAU,CAAC,KAAK,IAAI,KAAK,CAAC,CAC/D;EACA,IAAI,SAAwC;EAC5C,IAAI,SAAwC;EAC5C,IAAI,qBAAqB,KAAK,wBAAwB,KAAK,wBAAwB,GACjF,MAAM,IAAI,UAAU,mEAAmE;EAEzF,IAAI,MAAM,aAAa,WAAW,SAAS;OACtC,IACH,sBAAsB,MAAM,kBAC5B,yBAAyB,MAAM,eAE/B,SAAS;OACJ,IAAI,sBAAsB,MAAM,gBAAgB;GACrD,SAAS;GACT,SAAS;EACX,OAAO,IAAI,yBAAyB,MAAM,eACxC,IAAI,MAAM,aAAa,YAAY;GACjC,SAAS;GACT,SAAS;EACX,OAAO,SAAS;EAElB,OAAO;GACL;GACA;GACA;GACA;GACA;GACA;GACA,iBAAiB,OAAO,SAAS,SAAS,OAAO,OAAO;GACxD,OAAO,gBAAgB,KAAK;EAC9B;CACF,CAAC;CACD,OAAO;EACL,cAAc;EACd,gBAAgB,OAAO,QAAQ;EAC/B,aAAa,OAAO;EACpB;CACF;AACF;;AAGA,SAAgB,kBACd,QACA,QACA,SACkB;CAClB,IAAI,CAAC,QAAQ,UAAU,KAAK,GAAG,MAAM,IAAI,UAAU,wBAAwB;CAC3E,MAAM,SAAS,gBAAgB,QAAQ,MAAM;CAC7C,OAAO,OAAO,YACX,QAAQ,SAAS,KAAK,WAAW,SAAS,CAAC,CAC3C,KAAK,SACJ,YAAY;EACV,YAAY,QAAQ;EACpB,GAAI,QAAQ,eAAe,KAAA,IAAY,CAAC,IAAI,EAAE,aAAa,QAAQ,WAAW;EAC9E,SAAS,KAAK,MAAM;EACpB,MAAM,KAAK,WAAW,eAAe,wBAAwB,KAAK,MAAM;EACxE,UAAU,KAAK,WAAW,eAAe,SAAS,KAAK,MAAM;EAC7D,OACE,KAAK,WAAW,cACZ,+BAA+B,KAAK,MAAM,UAC1C,sBAAsB,KAAK,MAAM;EACvC,eAAe,gBAAgB,KAAK,MAAM,QAAQ;EAElD,YAAY,KAAK,WAAW,cAAc,KAAK,qBAAqB;EACpE,oBAAoB;EACpB,UAAU,cAAc;GAAC,KAAK;GAAU,KAAK;GAAQ,OAAO;GAAS,KAAK,MAAM;EAAK,CAAC;EACtF,UAAU;GACR,eAAe,OAAO;GACtB,UAAU,KAAK;GACf,iBAAiB,OAAO;GACxB,cAAc,OAAO;GACrB,YAAY,KAAK;GACjB,mBAAmB,KAAK;GACxB,mBAAmB,KAAK,MAAM;GAC9B,qBAAqB,KAAK;GAC1B,wBAAwB,KAAK;GAC7B,wBAAwB,KAAK;GAC7B,kBAAkB,KAAK;GACvB,aAAa;GACb,kBACE,KAAK,WAAW,cAAc,uBAAuB;GACvD,eAAe,gBAAgB,OAAO,QAAQ,KAAK,SAAS;EAC9D;CACF,CAAC,CACH;AACJ;;;;AC9MA,SAAgB,aAAa,SAA8B;CACzD,MAAM,SAAS;EAAE,GAAG;EAAS,SAAS,QAAQ,WAAW,EAAE,GAAG,QAAQ,QAAQ;CAAE;CAChF,MAAM,MAAM,gBAAqC;EAC/C,UAAU,SAAS,YAAY,OAAO,SAAS,gBAAgB,OAAO,GAAG,OAAO;EAChF,QAAQ,YAAY,QAAQ;EAC5B,YAAY,OAAO;EACnB,eAAe,OAAO;EACtB,kBAAkB,OAAO;EACzB,SACE,OAAO,aACL,QAAQ;GACR,MAAM,QAAQ,SAAS,GAAG;GAC1B,MAAM,EAAE,UAAU;GAClB,OAAO;IACL;IACA,aAAa,MAAM;IACnB,cAAc,MAAM;IACpB,oBAAoB,OAAO;GAC7B;EACF;EACF,WAAW,KAAK,YAAY;GAC1B,MAAM,SAAS,eAAe,KAAK,OAAO;GAC1C,IAAI,OAAO,eAAe,CAAC,OAAO,YAAY,QAAQ,OAAO,OAAO,KAAK,GACvE,MAAM,IAAI,MAAM,yDAAyD;EAE7E;CACF,CAAC;CACD,OAAO,OACL,SACA,YAC4C;EAC5C,SAAS,QAAQ,eAAe;EAChC,gBAAgB,OAAO;EACvB,MAAM,WAAW,gBAAgB,OAAO;EACxC,MAAM,SAAS,MAAM,IAAI,UAAU,OAAO;EAE1C,OAAO;GAAE,GAAG;GAAQ,OAAO,OAAO;EAAsB;CAC1D;AACF;;AA0BA,SAAS,iBAAiB,WAAyB;CACjD,OAAO;EACL,UAAU,aAAa,SAAS;EAChC,OAAO,OAAO,QAAQ,SAAS,CAAC,CAAC,KAAK,CAAC,MAAM,cAAc,CACzD,MACA,SAAS,SAAS,WAAW,OAAO,KAAK,SAAS,QAAQ,IAAI,IAChE,CAAC;CACH;AACF;AAEA,SAAS,gBAAgB,SAAiC,MAAsB;CAC9E,IACE,OAAO,QAAQ,OAAO,CAAC,CAAC,MACrB,CAAC,KAAK,YAAY,CAAC,KAAK,SAAS,GAAG,KAAK,CAAC,OAAO,SAAS,MAAM,KAAK,SAAS,CACjF,KACA,OAAO,OAAO,OAAO,CAAC,CAAC,QAAQ,KAAK,WAAW,MAAM,QAAQ,CAAC,KAAK,GAEnE,MAAM,IAAI,UAAU,uBAAuB;AAE/C;;AAGA,SAAgB,mBACd,QACA,SACY;CACZ,MAAM,aAAqC,OAAO,OAAO,IAAI;CAC7D,MAAM,eAAwD,OAAO,OAAO,IAAI;CAChF,KAAK,MAAM,CAAC,KAAK,WAAW,OAAO,QAAQ,OAAO,OAAO,GAAG;EAC1D,IAAI,OAAO,SAAS,UAAU,MAAM,IAAI,UAAU,yCAAyC;EAC3F,IAAI,OAAO,SAAS,QAAQ;GAC1B,WAAW,OAAO,OAAO;GACzB,aAAa,OAAO,CAClB;IAAE,OAAO;IAAG,aAAa,IAAI,OAAO;GAAK,GACzC;IAAE,OAAO;IAAG,aAAa,OAAO;GAAK,CACvC;EACF,OAAO;GACL,MAAM,QAAQ,OAAO,KAAK,OAAO,MAAM,CAAC,CAAC,SAAS;GAClD,WAAW,OAAO,OAAO,QAAQ;GACjC,aAAa,OAAO,OAAO,QAAQ,OAAO,aAAa,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB;IACtF,OAAO,OAAO,KAAK,IAAI;IACvB;GACF,EAAE;EACJ;CACF;CACA,MAAM,YAAY,WAAW,OAAO,YAAY,OAAO,KAAK,UAAU,CAAC,CAAC,KAAK,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;CAC9F,gBAAgB,WAAW,OAAO,KAAK,UAAU,CAAC;CAClD,OAAO;EACL;EACA,WAAW,kBAAkB;GAAE,MAAM;GAAY,SAAS;EAAU,CAAC,CAAC,CAAC;EACvE,OAAO;EACP,eAAe;EACf;CACF;AACF;;AAGA,SAAgB,SACd,MACA,SACmB;CACnB,MAAM,SAAS,EAAE,GAAG,QAAQ;CAC5B,MAAM,SACJ,OAAO,QAAQ,cAAc,aAAa,QAAQ,YAAY,gBAAgB,QAAQ,SAAS;CACjG,MAAM,UAAU,QAAQ,WAAW,EAAE,GAAG,QAAQ,QAAQ;CACxD,IAAI,CAAC,KAAK,KAAK,KAAK,CAAC,QAAQ,QAAQ,KAAK,GACxC,MAAM,IAAI,UAAU,qCAAqC;CAE3D,KAAK,OAAO,WAAW,cAAc,OAAO,QAAQ,CAAC,OAAO,YAC1D,MAAM,IAAI,UAAU,uEAAuE;CAE7F,IAAI,CAAC,OAAO,OAAO,OAAO,WAAW,YAAY;EAC/C,kBAAkB,MAAM;EACxB,IAAI,OAAO,OAAO,MAAM,CAAC,CAAC,MAAM,aAAa,SAAS,SAAS,QAAQ,GACrE,MAAM,IAAI,UACR,0EACF;EAEF,IAAI,SAAS,gBAAgB,SAAS,OAAO,KAAK,MAAM,CAAC;CAC3D;CACA,MAAM,aACJ,OAAO,cACP,OAAO,QAAQ,MAAM,CAAC,CAAC,KAAK,CAAC,KAAK,eAAe;EAC/C;EACA,aAAa,OAAO,SAAS,iBAAiB,WAAW,SAAS,eAAe;CACnF,EAAE;CACJ,MAAM,WAAW,aAAa,MAAM;CACpC,OAAO,QAA4B;EACjC;EACA,SAAS,YAAY;GACnB,OAAO,OAAO;GACd,SAAS,OAAO;GAChB,WAAW,OAAO,WAAW,aAAa,YAAY,iBAAiB,MAAM;GAC7E;GAEA,GAAI,UAAU,EAAE,QAAQ,IAAI,CAAC;EAC/B,CAAC;EACD;EACA,WAAW,OAAO;EAClB,UAAU,OAAO,OAAO,UAAU,CAAC,MACjC,SACE;GACE,OAAO,OAAO;GACd,OAAO,MAAM,OAAO,YAAY,OAAO,OAAO;GAC9C,WAAW,OAAO,WAAW,aAAa,MAAM,OAAO,KAAK,IAAI;EAClE,GACA,OACF;EACF,KAAK,OAAO,SAAS,UAAU,mBAAmB,OAAO,OAAO;EAChE,QAAQ,OAAO;CACjB,CAAC;AACH;AAuBA,SAAgB,WACd,SACY;CACZ,MAAM,SAAS,EAAE,GAAG,QAAQ;CAC5B,MAAM,SACJ,OAAO,QAAQ,cAAc,aAAa,QAAQ,YAAY,gBAAgB,QAAQ,SAAS;CACjG,IAAI,CAAC,OAAO,GAAG,KAAK,KAAK,CAAC,OAAO,QAAQ,KAAK,KAAK,CAAC,OAAO,YAAY,KAAK,GAC1E,MAAM,IAAI,UAAU,mDAAmD;CAEzE,MAAM,WAAW,aAAa,MAAM;CACpC,OAAO,UAA2B;EAChC,IAAI,OAAO;EACX,aAAa,OAAO;EACpB,WAAW,OAAO;EAClB,SAAS,YAAY;GACnB,OAAO,OAAO;GACd,SAAS,OAAO;GAChB,WAAW,OAAO,WAAW,aAAa,YAAY,iBAAiB,MAAM;EAC/E,CAAC;EACD,MAAM;GAAE,MAAM;GAAO,QAAQ,CAAC,OAAO,KAAK;EAAE;EAC5C,UAAU,OAAO,OAAO,SAAS,mBAC/B,SACE;GACE,OAAO,OAAO;GACd,OAAO,MAAM,OAAO,YAAY,OAAO,cAAc;GACrD,WAAW,OAAO,WAAW,aAAa,MAAM,OAAO,OAAO,cAAc,IAAI;EAClF,GACA,OACF;EACF,KAAK,OAAO;EACZ,QAAQ,OAAO;CACjB,CAAC;AACH"}
|
|
1
|
+
{"version":3,"file":"jev.js","names":[],"sources":["../src/jev.ts"],"sourcesContent":["import type { Analyst, AnalystContext, AnalystFinding } from './analyst/types'\nimport type { JudgeConfig, JudgeScore, Scenario } from './campaign/types'\nimport type { CostReceiptInput, CustomTokenPricing } from './cost-ledger'\nimport type { EvaluationContext, EvaluationResult, EvaluatorOptions } from './evaluation'\nimport { asAnalyst, asJudge, createEvaluator } from './evaluation'\nimport type { JevQuestions, JevRequest, JevResult, JevState } from './jev-protocol'\nimport { jevUsage, parseJevQuestions, parseJevRequest, parseJevResult } from './jev-protocol'\nimport { jsonDocument } from './ledger-core/canonical'\nimport { weightedComposite } from './statistics'\nimport { contentHash } from './verdict-cache'\n\nexport * from './evaluation'\nexport * from './jev-protocol'\nexport * from './jev-review'\n\nexport type JevEvaluate = EvaluatorOptions<JevRequest, unknown>['execute']\nexport interface JevEvaluatorOptions {\n evaluate: JevEvaluate\n costLedger?: EvaluatorOptions<JevRequest, unknown>['costLedger']\n maximumCharge?: EvaluatorOptions<JevRequest, unknown>['maximumCharge']\n pricing?: CustomTokenPricing\n receipt?: (response: unknown) => CostReceiptInput\n receiptFromError?: (error: Error) => CostReceiptInput | undefined\n /** Optional deployment policy; the protocol adapter does not hardcode model aliases. */\n acceptModel?: (requested: string, served: string) => boolean\n}\n\n/** Configure transport once; supply native state, questions, and model on every invocation. */\nexport function jevEvaluator(options: JevEvaluatorOptions) {\n const config = { ...options, pricing: options.pricing && { ...options.pricing } }\n const run = createEvaluator<JevRequest, unknown>({\n execute: (request, context) => config.evaluate(structuredClone(request), context),\n model: (request) => request.model,\n costLedger: config.costLedger,\n maximumCharge: config.maximumCharge,\n receiptFromError: config.receiptFromError,\n receipt:\n config.receipt ??\n ((raw) => {\n const usage = jevUsage(raw)\n const { model } = raw as JevResult\n return {\n model,\n inputTokens: usage.input_tokens,\n outputTokens: usage.output_tokens,\n customTokenPricing: config.pricing,\n }\n }),\n validate: (raw, request) => {\n const result = parseJevResult(raw, request)\n if (config.acceptModel && !config.acceptModel(request.model, result.model)) {\n throw new Error('Evaluation served a model rejected by the caller policy')\n }\n },\n })\n return async <const Q extends JevQuestions>(\n request: JevRequest<Q>,\n context?: EvaluationContext,\n ): Promise<EvaluationResult<JevResult<Q>>> => {\n context?.signal?.throwIfAborted()\n parseJevRequest(request)\n const snapshot = structuredClone(request)\n const result = await run(snapshot, context)\n // The validator checked these exact questions after recording the paid work.\n return { ...result, value: result.value as JevResult<Q> }\n }\n}\n\ntype Input<A, S extends Scenario> = { artifact: A; scenario: S }\ntype QuestionSource<I, Q extends JevQuestions> = Q | ((input: I) => Q | Promise<Q>)\nexport interface JevOptions extends JevEvaluatorOptions {\n model: string\n version: string\n questions: JevQuestions\n}\nexport interface JevJudgeOptions<\n A,\n S extends Scenario = Scenario,\n Q extends JevQuestions = JevQuestions,\n> extends JevEvaluatorOptions {\n model: string\n version: string\n questions: QuestionSource<Input<A, S>, Q>\n renderState: (input: Input<A, S>, context: EvaluationContext) => JevState | Promise<JevState>\n dimensions?: JudgeConfig<A, S>['dimensions']\n weights?: Record<string, number>\n map?: (value: JevResult<Q>, input: Input<A, S>) => JudgeScore | Promise<JudgeScore>\n record?: (result: EvaluationResult<JevResult<Q>>, input: Input<A, S>) => void | Promise<void>\n appliesTo?: (scenario: S) => boolean\n}\n\n/** Canonical identity also retains native question/alternative order; dispatch stays untouched. */\nfunction questionIdentity(questions: JevQuestions) {\n return {\n document: jsonDocument(questions),\n order: Object.entries(questions).map(([name, question]) => [\n name,\n question.type === 'choice' ? Object.keys(question.criteria) : null,\n ]),\n }\n}\n\nfunction validateWeights(weights: Record<string, number>, keys: string[]): void {\n if (\n Object.entries(weights).some(\n ([key, weight]) => !keys.includes(key) || !Number.isFinite(weight) || weight < 0,\n ) ||\n Object.values(weights).reduce((sum, weight) => sum + weight, 0) <= 0\n ) {\n throw new TypeError('Invalid judge weights')\n }\n}\n\n/** Optional convention. Choice utilities and non-uniform scales belong in an explicit map. */\nexport function normalizedJevScore(\n result: JevResult,\n weights?: Record<string, number>,\n): JudgeScore {\n const dimensions: Record<string, number> = Object.create(null)\n const distribution: NonNullable<JudgeScore['distribution']> = Object.create(null)\n for (const [key, answer] of Object.entries(result.answers)) {\n if (answer.type === 'choice') throw new TypeError('Choice scoring requires an explicit map')\n if (answer.type === 'noul') {\n dimensions[key] = answer.noul\n distribution[key] = [\n { score: 0, probability: 1 - answer.noul },\n { score: 1, probability: answer.noul },\n ]\n } else {\n const scale = Object.keys(answer.legend).length - 1\n dimensions[key] = answer.score / scale\n distribution[key] = Object.entries(answer.probabilities).map(([level, probability]) => ({\n score: Number(level) / scale,\n probability,\n }))\n }\n }\n const effective = weights ?? Object.fromEntries(Object.keys(dimensions).map((key) => [key, 1]))\n validateWeights(effective, Object.keys(dimensions))\n return {\n dimensions,\n composite: weightedComposite({ dims: dimensions, weights: effective }).composite,\n notes: '',\n scoringMethod: 'expectation',\n distribution,\n }\n}\n\n/** Convenience around the public evaluator. No domain questions, thresholds, or prose. */\nexport function jevJudge<A, S extends Scenario = Scenario, Q extends JevQuestions = JevQuestions>(\n name: string,\n options: JevJudgeOptions<A, S, Q>,\n): JudgeConfig<A, S> {\n const config = { ...options }\n const source =\n typeof options.questions === 'function' ? options.questions : structuredClone(options.questions)\n const weights = options.weights && { ...options.weights }\n if (!name.trim() || !options.version.trim()) {\n throw new TypeError('Judge name and version are required')\n }\n if ((typeof source === 'function' || config.map) && !config.dimensions) {\n throw new TypeError('Dynamic questions and custom mappings require stable judge dimensions')\n }\n if (!config.map && typeof source !== 'function') {\n parseJevQuestions(source)\n if (Object.values(source).some((question) => question.type === 'choice')) {\n throw new TypeError(\n 'Choice scoring requires an explicit map; labels have no numeric ordering',\n )\n }\n if (weights) validateWeights(weights, Object.keys(source))\n }\n const dimensions =\n config.dimensions ??\n Object.entries(source).map(([key, question]) => ({\n key,\n description: typeof question.instructions === 'string' ? question.instructions : key,\n }))\n const evaluate = jevEvaluator(config)\n return asJudge<A, S, JevResult<Q>>({\n name,\n version: contentHash({\n model: config.model,\n version: config.version,\n questions: typeof source === 'function' ? 'dynamic' : questionIdentity(source),\n dimensions,\n // Hash the JSON document: optional undefined fields are absent on the wire.\n ...(weights ? { weights } : {}),\n }),\n dimensions,\n appliesTo: config.appliesTo,\n evaluate: async (input, context = {}) =>\n evaluate(\n {\n model: config.model,\n state: await config.renderState(input, context),\n questions: typeof source === 'function' ? await source(input) : source,\n },\n context,\n ),\n map: config.map ?? ((value) => normalizedJevScore(value, weights)),\n record: config.record,\n })\n}\n\nexport interface JevAnalystOptions<I, Q extends JevQuestions = JevQuestions>\n extends JevEvaluatorOptions {\n id: string\n description: string\n inputKind: Analyst<I>['inputKind']\n model: string\n version: string\n questions: Q | ((input: I, context: AnalystContext) => Q | Promise<Q>)\n renderState: (input: I, context: AnalystContext) => JevState | Promise<JevState>\n findings: (\n value: JevResult<Q>,\n input: I,\n context: AnalystContext,\n ) => AnalystFinding[] | Promise<AnalystFinding[]>\n record?: (\n result: EvaluationResult<JevResult<Q>>,\n input: I,\n context: AnalystContext,\n ) => void | Promise<void>\n}\n\nexport function jevAnalyst<I, Q extends JevQuestions = JevQuestions>(\n options: JevAnalystOptions<I, Q>,\n): Analyst<I> {\n const config = { ...options }\n const source =\n typeof options.questions === 'function' ? options.questions : structuredClone(options.questions)\n if (!config.id.trim() || !config.version.trim() || !config.description.trim()) {\n throw new TypeError('Analyst id, version, and description are required')\n }\n const evaluate = jevEvaluator(config)\n return asAnalyst<I, JevResult<Q>>({\n id: config.id,\n description: config.description,\n inputKind: config.inputKind,\n version: contentHash({\n model: config.model,\n version: config.version,\n questions: typeof source === 'function' ? 'dynamic' : questionIdentity(source),\n }),\n cost: { kind: 'llm', models: [config.model] },\n evaluate: async (input, context, analystContext) =>\n evaluate(\n {\n model: config.model,\n state: await config.renderState(input, analystContext),\n questions: typeof source === 'function' ? await source(input, analystContext) : source,\n },\n context,\n ),\n map: config.findings,\n record: config.record,\n })\n}\n"],"mappings":";;;;;;;;AA4BA,SAAgB,aAAa,SAA8B;CACzD,MAAM,SAAS;EAAE,GAAG;EAAS,SAAS,QAAQ,WAAW,EAAE,GAAG,QAAQ,QAAQ;CAAE;CAChF,MAAM,MAAM,gBAAqC;EAC/C,UAAU,SAAS,YAAY,OAAO,SAAS,gBAAgB,OAAO,GAAG,OAAO;EAChF,QAAQ,YAAY,QAAQ;EAC5B,YAAY,OAAO;EACnB,eAAe,OAAO;EACtB,kBAAkB,OAAO;EACzB,SACE,OAAO,aACL,QAAQ;GACR,MAAM,QAAQ,SAAS,GAAG;GAC1B,MAAM,EAAE,UAAU;GAClB,OAAO;IACL;IACA,aAAa,MAAM;IACnB,cAAc,MAAM;IACpB,oBAAoB,OAAO;GAC7B;EACF;EACF,WAAW,KAAK,YAAY;GAC1B,MAAM,SAAS,eAAe,KAAK,OAAO;GAC1C,IAAI,OAAO,eAAe,CAAC,OAAO,YAAY,QAAQ,OAAO,OAAO,KAAK,GACvE,MAAM,IAAI,MAAM,yDAAyD;EAE7E;CACF,CAAC;CACD,OAAO,OACL,SACA,YAC4C;EAC5C,SAAS,QAAQ,eAAe;EAChC,gBAAgB,OAAO;EACvB,MAAM,WAAW,gBAAgB,OAAO;EACxC,MAAM,SAAS,MAAM,IAAI,UAAU,OAAO;EAE1C,OAAO;GAAE,GAAG;GAAQ,OAAO,OAAO;EAAsB;CAC1D;AACF;;AA0BA,SAAS,iBAAiB,WAAyB;CACjD,OAAO;EACL,UAAU,aAAa,SAAS;EAChC,OAAO,OAAO,QAAQ,SAAS,CAAC,CAAC,KAAK,CAAC,MAAM,cAAc,CACzD,MACA,SAAS,SAAS,WAAW,OAAO,KAAK,SAAS,QAAQ,IAAI,IAChE,CAAC;CACH;AACF;AAEA,SAAS,gBAAgB,SAAiC,MAAsB;CAC9E,IACE,OAAO,QAAQ,OAAO,CAAC,CAAC,MACrB,CAAC,KAAK,YAAY,CAAC,KAAK,SAAS,GAAG,KAAK,CAAC,OAAO,SAAS,MAAM,KAAK,SAAS,CACjF,KACA,OAAO,OAAO,OAAO,CAAC,CAAC,QAAQ,KAAK,WAAW,MAAM,QAAQ,CAAC,KAAK,GAEnE,MAAM,IAAI,UAAU,uBAAuB;AAE/C;;AAGA,SAAgB,mBACd,QACA,SACY;CACZ,MAAM,aAAqC,OAAO,OAAO,IAAI;CAC7D,MAAM,eAAwD,OAAO,OAAO,IAAI;CAChF,KAAK,MAAM,CAAC,KAAK,WAAW,OAAO,QAAQ,OAAO,OAAO,GAAG;EAC1D,IAAI,OAAO,SAAS,UAAU,MAAM,IAAI,UAAU,yCAAyC;EAC3F,IAAI,OAAO,SAAS,QAAQ;GAC1B,WAAW,OAAO,OAAO;GACzB,aAAa,OAAO,CAClB;IAAE,OAAO;IAAG,aAAa,IAAI,OAAO;GAAK,GACzC;IAAE,OAAO;IAAG,aAAa,OAAO;GAAK,CACvC;EACF,OAAO;GACL,MAAM,QAAQ,OAAO,KAAK,OAAO,MAAM,CAAC,CAAC,SAAS;GAClD,WAAW,OAAO,OAAO,QAAQ;GACjC,aAAa,OAAO,OAAO,QAAQ,OAAO,aAAa,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB;IACtF,OAAO,OAAO,KAAK,IAAI;IACvB;GACF,EAAE;EACJ;CACF;CACA,MAAM,YAAY,WAAW,OAAO,YAAY,OAAO,KAAK,UAAU,CAAC,CAAC,KAAK,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;CAC9F,gBAAgB,WAAW,OAAO,KAAK,UAAU,CAAC;CAClD,OAAO;EACL;EACA,WAAW,kBAAkB;GAAE,MAAM;GAAY,SAAS;EAAU,CAAC,CAAC,CAAC;EACvE,OAAO;EACP,eAAe;EACf;CACF;AACF;;AAGA,SAAgB,SACd,MACA,SACmB;CACnB,MAAM,SAAS,EAAE,GAAG,QAAQ;CAC5B,MAAM,SACJ,OAAO,QAAQ,cAAc,aAAa,QAAQ,YAAY,gBAAgB,QAAQ,SAAS;CACjG,MAAM,UAAU,QAAQ,WAAW,EAAE,GAAG,QAAQ,QAAQ;CACxD,IAAI,CAAC,KAAK,KAAK,KAAK,CAAC,QAAQ,QAAQ,KAAK,GACxC,MAAM,IAAI,UAAU,qCAAqC;CAE3D,KAAK,OAAO,WAAW,cAAc,OAAO,QAAQ,CAAC,OAAO,YAC1D,MAAM,IAAI,UAAU,uEAAuE;CAE7F,IAAI,CAAC,OAAO,OAAO,OAAO,WAAW,YAAY;EAC/C,kBAAkB,MAAM;EACxB,IAAI,OAAO,OAAO,MAAM,CAAC,CAAC,MAAM,aAAa,SAAS,SAAS,QAAQ,GACrE,MAAM,IAAI,UACR,0EACF;EAEF,IAAI,SAAS,gBAAgB,SAAS,OAAO,KAAK,MAAM,CAAC;CAC3D;CACA,MAAM,aACJ,OAAO,cACP,OAAO,QAAQ,MAAM,CAAC,CAAC,KAAK,CAAC,KAAK,eAAe;EAC/C;EACA,aAAa,OAAO,SAAS,iBAAiB,WAAW,SAAS,eAAe;CACnF,EAAE;CACJ,MAAM,WAAW,aAAa,MAAM;CACpC,OAAO,QAA4B;EACjC;EACA,SAAS,YAAY;GACnB,OAAO,OAAO;GACd,SAAS,OAAO;GAChB,WAAW,OAAO,WAAW,aAAa,YAAY,iBAAiB,MAAM;GAC7E;GAEA,GAAI,UAAU,EAAE,QAAQ,IAAI,CAAC;EAC/B,CAAC;EACD;EACA,WAAW,OAAO;EAClB,UAAU,OAAO,OAAO,UAAU,CAAC,MACjC,SACE;GACE,OAAO,OAAO;GACd,OAAO,MAAM,OAAO,YAAY,OAAO,OAAO;GAC9C,WAAW,OAAO,WAAW,aAAa,MAAM,OAAO,KAAK,IAAI;EAClE,GACA,OACF;EACF,KAAK,OAAO,SAAS,UAAU,mBAAmB,OAAO,OAAO;EAChE,QAAQ,OAAO;CACjB,CAAC;AACH;AAuBA,SAAgB,WACd,SACY;CACZ,MAAM,SAAS,EAAE,GAAG,QAAQ;CAC5B,MAAM,SACJ,OAAO,QAAQ,cAAc,aAAa,QAAQ,YAAY,gBAAgB,QAAQ,SAAS;CACjG,IAAI,CAAC,OAAO,GAAG,KAAK,KAAK,CAAC,OAAO,QAAQ,KAAK,KAAK,CAAC,OAAO,YAAY,KAAK,GAC1E,MAAM,IAAI,UAAU,mDAAmD;CAEzE,MAAM,WAAW,aAAa,MAAM;CACpC,OAAO,UAA2B;EAChC,IAAI,OAAO;EACX,aAAa,OAAO;EACpB,WAAW,OAAO;EAClB,SAAS,YAAY;GACnB,OAAO,OAAO;GACd,SAAS,OAAO;GAChB,WAAW,OAAO,WAAW,aAAa,YAAY,iBAAiB,MAAM;EAC/E,CAAC;EACD,MAAM;GAAE,MAAM;GAAO,QAAQ,CAAC,OAAO,KAAK;EAAE;EAC5C,UAAU,OAAO,OAAO,SAAS,mBAC/B,SACE;GACE,OAAO,OAAO;GACd,OAAO,MAAM,OAAO,YAAY,OAAO,cAAc;GACrD,WAAW,OAAO,WAAW,aAAa,MAAM,OAAO,OAAO,cAAc,IAAI;EAClF,GACA,OACF;EACF,KAAK,OAAO;EACZ,QAAQ,OAAO;CACjB,CAAC;AACH"}
|
|
@@ -16,9 +16,9 @@ import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "./cla
|
|
|
16
16
|
import { f as redactText, n as REDACTION_VERSION, o as detectCredential, s as detectPersonalData } from "./redact-Dk1ZdJEm.js";
|
|
17
17
|
import { t as detectRewardHacking } from "./reward-hacking-D0XwhVWE.js";
|
|
18
18
|
import { n as assertProposalFindings, t as combineAbortSignals } from "./abort-signal-CtzAM_sJ.js";
|
|
19
|
-
import { D as paretoFrontier, O as paretoFrontierWithCrowding, a as searchPolicyView, b as estimateNode, i as searchExpansionIndex, n as runSearch, p as EXPANSION_OPERATORS, t as SEARCH_KERNEL_SOURCE } from "./search-kernel-
|
|
20
|
-
import { a as validateSearchLedgerEvent, d as searchTaskSetDigest, i as replaySearchLedgerText, l as searchCellId, n as openSearchLedger, u as searchNodeId } from "./search-ledger-
|
|
21
|
-
import { n as searchProposerView, t as renderSearchSummary } from "./search-summary-
|
|
19
|
+
import { D as paretoFrontier, O as paretoFrontierWithCrowding, a as searchPolicyView, b as estimateNode, i as searchExpansionIndex, n as runSearch, p as EXPANSION_OPERATORS, t as SEARCH_KERNEL_SOURCE } from "./search-kernel-AVgIjXaI.js";
|
|
20
|
+
import { a as validateSearchLedgerEvent, d as searchTaskSetDigest, i as replaySearchLedgerText, l as searchCellId, n as openSearchLedger, u as searchNodeId } from "./search-ledger-CL0-mXQ8.js";
|
|
21
|
+
import { n as searchProposerView, t as renderSearchSummary } from "./search-summary-DvWk0hb3.js";
|
|
22
22
|
import { d as stripFencedJson, o as costReceiptFromLlm, s as costReceiptFromLlmError, u as maximumChargeForLlmRequest } from "./llm-client-c_bKHTXz.js";
|
|
23
23
|
import { createHash } from "node:crypto";
|
|
24
24
|
import { writeFileSync } from "node:fs";
|
|
@@ -6127,4 +6127,4 @@ function firstString(value) {
|
|
|
6127
6127
|
//#endregion
|
|
6128
6128
|
export { captureFinalEvidencePolicy as $, assertCompleteSearchHistory as A, searchReceiptAccounting as B, incumbentWithOperatorBandit as C, surfaceDiff as D, searchEdgeId as E, verifySearchHistoryArtifact as F, captureJudge as G, uniform as H, verifySearchHistoryReceipt as I, redTeamDataset as J, defaultProductionGate as K, campaignCellSearchResult as L, assertSearchHistoryMatchesState as M, createSearchHistoryReceipt as N, surfaceNode as O, searchHistoryCoverageRow as P, assertIndependentEvaluationSplit as Q, searchModelIdentity as R, incumbent as S, developmentClaim as T, runFinalComparison as U, asha as V, openAutoPr as W, scoreRedTeamOutput as X, redTeamReport as Y, runCanaries as Z, isProposedCandidate as _, transientDispatchFailure as a, resolveRunDir as at, beam as b, compareOptimizationMethods as c, buildCellSchedule as ct, combineComparisonCosts as d, evaluationUnitMap as et, costFromLedgerSummary as f, runOptimization as g, runImprovementLoop as h, quotaExhaustedUntil as i, planCampaignRun as it, assertSearchHistoryAdmissionOptions as j, SearchHistoryRequiredError as k, optimizationTokenUsageFromSummary as l, cellCachePath as lt, readGepaCandidatePopulationArtifact as m, JudgeParseError as n, runEval as nt, aggregateRunScore as o, tangleTracesRoot as ot, assertGepaCandidatePopulationSummary as p, DEFAULT_RED_TEAM_CORPUS as q, isTransientTransportFailure as r, runCampaign as rt, clamp01 as s, readCachedCell as st, llmJudge as t, reserveFinalEvidence as tt, executeOptimizationMethod as u, computeManifestHash as ut, labelTrustRank as v, SearchRecorder as w, crowdedFrontierParent as x, aide as y, searchProposalExecution as z };
|
|
6129
6129
|
|
|
6130
|
-
//# sourceMappingURL=llm-judge-
|
|
6130
|
+
//# sourceMappingURL=llm-judge-CL6f3utD.js.map
|