@tangle-network/agent-eval 0.205.2 → 0.207.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/dist/adapters/http.d.ts +1 -1
- package/dist/analyst/index.d.ts +1 -1
- package/dist/analyst/index.js +1 -1
- package/dist/{benchmark-command-DuDTN5ID.js → benchmark-command-DtP94cOB.js} +2 -2
- package/dist/{benchmark-command-DuDTN5ID.js.map → benchmark-command-DtP94cOB.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +2 -2
- package/dist/campaign/index.d.ts +3 -3
- package/dist/campaign/index.js +2 -2
- package/dist/cli.js +3 -2
- package/dist/cli.js.map +1 -1
- package/dist/contract/index.d.ts +4 -4
- package/dist/{define-agent-eval-CKB6wyML.d.ts → define-agent-eval-B2YV-RDE.d.ts} +3 -3
- package/dist/{define-agent-eval-CKB6wyML.d.ts.map → define-agent-eval-B2YV-RDE.d.ts.map} +1 -1
- package/dist/{index-BRdZ0oJo.d.ts → index-CojsA_EE.d.ts} +2 -2
- package/dist/{index-BRdZ0oJo.d.ts.map → index-CojsA_EE.d.ts.map} +1 -1
- package/dist/{index-R8Xhod8e.d.ts → index-DVDiJIhi.d.ts} +3 -3
- package/dist/{index-R8Xhod8e.d.ts.map → index-DVDiJIhi.d.ts.map} +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/meta-eval/index.d.ts +69 -0
- package/dist/meta-eval/index.d.ts.map +1 -1
- package/dist/meta-eval/index.js +135 -1
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/rl.d.ts +12 -12
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +17 -17
- package/dist/rl.js.map +1 -1
- package/dist/search/index.d.ts +8 -2
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +1 -1
- package/dist/{search-kernel-DNpsHb8P.d.ts → search-kernel-CZ8KJQRw.d.ts} +6 -3
- package/dist/{search-kernel-DNpsHb8P.d.ts.map → search-kernel-CZ8KJQRw.d.ts.map} +1 -1
- package/dist/systemone-protocol-DKoi1NdK.d.ts +132 -0
- package/dist/systemone-protocol-DKoi1NdK.d.ts.map +1 -0
- package/dist/systemone-protocol.d.ts +2 -0
- package/dist/systemone-protocol.js +214 -0
- package/dist/systemone-protocol.js.map +1 -0
- package/dist/{jev-review-Cw06AKuF.d.ts → systemone-review-DGTKkLMG.d.ts} +16 -16
- package/dist/systemone-review-DGTKkLMG.d.ts.map +1 -0
- package/dist/{jev-review-DZEjDzlF.js → systemone-review-DTClwgmE.js} +11 -11
- package/dist/systemone-review-DTClwgmE.js.map +1 -0
- package/dist/systemone.d.ts +60 -0
- package/dist/systemone.d.ts.map +1 -0
- package/dist/{jev.js → systemone.js} +16 -16
- package/dist/systemone.js.map +1 -0
- package/dist/{tree-By7LtLUu.js → tree-CN8mlzUO.js} +11 -3
- package/dist/{tree-By7LtLUu.js.map → tree-CN8mlzUO.js.map} +1 -1
- package/docs/changes/jev-evidence-review.md +1 -1
- package/docs/changes/systemone-provider-neutral.md +36 -0
- package/docs/public-api.md +96 -70
- package/docs/search-ledger.md +9 -3
- package/docs/{jev-behavior-review.md → systemone-behavior-review.md} +17 -17
- package/docs/{jev-integrations.md → systemone-integrations.md} +4 -4
- package/docs/{jev-replay.md → systemone-replay.md} +6 -6
- package/docs/{jev.md → systemone.md} +96 -33
- package/package.json +9 -9
- package/dist/jev-protocol-BQLxwbqD.d.ts +0 -99
- package/dist/jev-protocol-BQLxwbqD.d.ts.map +0 -1
- package/dist/jev-protocol.d.ts +0 -2
- package/dist/jev-protocol.js +0 -114
- package/dist/jev-protocol.js.map +0 -1
- package/dist/jev-review-Cw06AKuF.d.ts.map +0 -1
- package/dist/jev-review-DZEjDzlF.js.map +0 -1
- package/dist/jev.d.ts +0 -60
- package/dist/jev.d.ts.map +0 -1
- package/dist/jev.js.map +0 -1
package/dist/rl.d.ts
CHANGED
|
@@ -13,8 +13,8 @@ import { t as InterimReleaseConfidence } from "./sequential-BhsrMupG.js";
|
|
|
13
13
|
import { t as AdversarialMutation } from "./adversarial-smnADNFS.js";
|
|
14
14
|
import { a as SteeringChange, f as EvalCampaignOptions, h as runEvalCampaign, i as Researcher, n as ExperimentResult, p as EvalCampaignResult, r as FailureMode, t as ExperimentPlan } from "./researcher-zTpxNJBy.js";
|
|
15
15
|
import { o as TrainingExportOptions } from "./exporters-ChTnR90l.js";
|
|
16
|
-
import { a as PreparedJevReview, i as JevReviewReport } from "./jev-review-Cw06AKuF.js";
|
|
17
16
|
import { a as RubricPredictiveValidityReport, t as OutcomeMetricSpec } from "./rubric-predictive-validity-DIUwJJ9Y.js";
|
|
17
|
+
import { a as SystemOneReviewReport, t as PreparedSystemOneReview } from "./systemone-review-DGTKkLMG.js";
|
|
18
18
|
//#region src/rl/active-curriculum.d.ts
|
|
19
19
|
/**
|
|
20
20
|
* Adaptive curriculum: active scenario selection by variance.
|
|
@@ -229,11 +229,11 @@ export interface IntegrityClaim {
|
|
|
229
229
|
readonly novelty: 'new' | 'known' | null;
|
|
230
230
|
/** The exact checker's verdict, such as `verified-new`. */
|
|
231
231
|
readonly checkerVerdict: string;
|
|
232
|
-
/** The claim page. Read by the text signal only; never sent to a referee or to
|
|
232
|
+
/** The claim page. Read by the text signal only; never sent to a referee or to a System One model. */
|
|
233
233
|
readonly text?: string | null;
|
|
234
234
|
}
|
|
235
235
|
export type IntegritySignalKind = 'exact' | 'anomaly' | 'text' | 'model' | 'context';
|
|
236
|
-
export type IntegritySignalId = 'scope-declared' | 'excluded-case' | 'listed-in-source' | 'inside-checked-range' | 'parameter-farming' | 'grader-reference' | 'referee' | '
|
|
236
|
+
export type IntegritySignalId = 'scope-declared' | 'excluded-case' | 'listed-in-source' | 'inside-checked-range' | 'parameter-farming' | 'grader-reference' | 'referee' | 'systemone' | 'trajectory-monitor' | 'checker-read-first';
|
|
237
237
|
export interface IntegrityEvidence {
|
|
238
238
|
readonly kind: 'source' | 'claim' | 'lane' | 'trace' | 'model';
|
|
239
239
|
readonly ref: string;
|
|
@@ -310,7 +310,7 @@ export declare function checkerReadSignal(events: readonly TrajectoryEvent[], op
|
|
|
310
310
|
checkers: readonly string[];
|
|
311
311
|
firstClaimAt: string | number | null;
|
|
312
312
|
}): IntegritySignal;
|
|
313
|
-
/** What the referee and
|
|
313
|
+
/** What the referee and the System One model read about a claim: the source and the claim's parameters, never its prose. */
|
|
314
314
|
export declare function claimBrief(claim: IntegrityClaim, scope: SourceScope): {
|
|
315
315
|
statement: string;
|
|
316
316
|
source: string;
|
|
@@ -358,10 +358,10 @@ export declare function refereeSignal(claim: IntegrityClaim, answer: SourceScope
|
|
|
358
358
|
costUsd?: number | null;
|
|
359
359
|
}): IntegritySignal;
|
|
360
360
|
/**
|
|
361
|
-
*
|
|
361
|
+
* System One typed questions over the same brief the referee reads. Each names a way the claim can be true of
|
|
362
362
|
* the checker and not of the statement; `true` supports that hypothesis, except `inScope`.
|
|
363
363
|
*/
|
|
364
|
-
export declare const
|
|
364
|
+
export declare const claimIntegritySystemOneQuestions: {
|
|
365
365
|
readonly excludedBySource: {
|
|
366
366
|
readonly type: 'noul';
|
|
367
367
|
readonly instructions: "Using only the source excerpts, does the source exclude the claim's case from its statement, by a hypothesis, a definition, or an assumption it states (for example a proof that calls a case easy and then assumes it away)?";
|
|
@@ -379,8 +379,8 @@ export declare const claimIntegrityJevQuestions: {
|
|
|
379
379
|
readonly instructions: "Is the claim's case inside the statement the source means, so that a verified result there would count for the source's authors?";
|
|
380
380
|
};
|
|
381
381
|
};
|
|
382
|
-
export type
|
|
383
|
-
export interface
|
|
382
|
+
export type ClaimIntegritySystemOneQuestions = typeof claimIntegritySystemOneQuestions;
|
|
383
|
+
export interface ClaimIntegritySystemOneOptions {
|
|
384
384
|
readonly model: string;
|
|
385
385
|
/**
|
|
386
386
|
* Support mass at or above which a question counts against the claim. The caller calibrates it:
|
|
@@ -391,10 +391,10 @@ export interface ClaimIntegrityJevOptions {
|
|
|
391
391
|
readonly refuteAtLeast: number;
|
|
392
392
|
readonly version?: string;
|
|
393
393
|
}
|
|
394
|
-
/** A prepared
|
|
395
|
-
export declare function
|
|
396
|
-
/**
|
|
397
|
-
export declare function
|
|
394
|
+
/** A prepared System One review of one claim, for `systemOneEvaluator` or any transport; assess with `assessSystemOneReview`. */
|
|
395
|
+
export declare function claimIntegritySystemOneReview(claim: IntegrityClaim, scope: SourceScope, options: ClaimIntegritySystemOneOptions): PreparedSystemOneReview<ClaimIntegritySystemOneQuestions>;
|
|
396
|
+
/** The System One signal from an assessed review: any question supported against the claim fires it. */
|
|
397
|
+
export declare function systemOneSignal(report: SystemOneReviewReport | {
|
|
398
398
|
readonly error: string;
|
|
399
399
|
} | null, options?: {
|
|
400
400
|
costUsd?: number | null;
|
package/dist/rl.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rl.d.ts","names":[],"sources":["../src/rl/active-curriculum.ts","../src/rl/adaptation-eval.ts","../src/rl/claim-integrity.ts","../src/rl/compute-curves.ts","../src/rl/contamination.ts","../src/rl/rollout-input.ts","../src/rl/preferences.ts","../src/rl/process-reward.ts","../src/rl/exporters.ts","../src/rl/dataset.ts","../src/rl/corpus.ts","../src/rl/off-policy.ts","../src/rl/predictive-validity-researcher.ts","../src/rl/rl-campaign.ts","../src/rl/run-record-adapters.ts","../src/rl/sim-fidelity.ts","../src/rl/tournament.ts","../src/rl/verified-findings-dataset.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAgBiB;EACf;EACA;;EAEA;;iBAGe;EACf;EACA;;EAEA;;EAEA;;iBAGe;;EAEf;;;;;;EAMA;;;;;EAKA;;;;;;;;;wBAUc,wBACd,cAAc,mBACd,gBAAgB;EAAQ;EAAmB;IAC3C,MAAM,4BACL;;;iBC9Cc,iBAAiB;;;;;;EAMhC,IAAI;IAAQ,UAAU;IAAG;IAAW;MAAgB;;iBAGrC,0BAA0B;EACzC,WAAW;;EAEX;;EAEA;EACA,QAAQ,iBAAiB;;EAEzB;;iBAGe;EACf;EACA;EACA;EACA;EACA;;EAEA,aAAa;IAAQ;IAAoB;IAAmB;IAAgB;;;iBAG7D;EACf,QAAQ;;;;;EAKR;;;;;EAKA;;wBAGoB,mBAAmB;EAAY;GACnD,MAAM,0BAA0B,KAC/B,QAAQ;iBAmEM;EACf,MAAM;IACJ;;IAEA,OAAO;;;EAGT,WAAW;;EAEX,cAAc;EACd,cAAc;;EAEd;;EAEA;;EAEA;;;;;;;;;wBAUc,wBACd,GAAG,iBACH,GAAG,iBACH;EACE;EACA;EACA;;EAEA;IAED;;wBA0Ea,WAAW,OAAO,iBAAiB;;;
|
|
1
|
+
{"version":3,"file":"rl.d.ts","names":[],"sources":["../src/rl/active-curriculum.ts","../src/rl/adaptation-eval.ts","../src/rl/claim-integrity.ts","../src/rl/compute-curves.ts","../src/rl/contamination.ts","../src/rl/rollout-input.ts","../src/rl/preferences.ts","../src/rl/process-reward.ts","../src/rl/exporters.ts","../src/rl/dataset.ts","../src/rl/corpus.ts","../src/rl/off-policy.ts","../src/rl/predictive-validity-researcher.ts","../src/rl/rl-campaign.ts","../src/rl/run-record-adapters.ts","../src/rl/sim-fidelity.ts","../src/rl/tournament.ts","../src/rl/verified-findings-dataset.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAgBiB;EACf;EACA;;EAEA;;iBAGe;EACf;EACA;;EAEA;;EAEA;;iBAGe;;EAEf;;;;;;EAMA;;;;;EAKA;;;;;;;;;wBAUc,wBACd,cAAc,mBACd,gBAAgB;EAAQ;EAAmB;IAC3C,MAAM,4BACL;;;iBC9Cc,iBAAiB;;;;;;EAMhC,IAAI;IAAQ,UAAU;IAAG;IAAW;MAAgB;;iBAGrC,0BAA0B;EACzC,WAAW;;EAEX;;EAEA;EACA,QAAQ,iBAAiB;;EAEzB;;iBAGe;EACf;EACA;EACA;EACA;EACA;;EAEA,aAAa;IAAQ;IAAoB;IAAmB;IAAgB;;;iBAG7D;EACf,QAAQ;;;;;EAKR;;;;;EAKA;;wBAGoB,mBAAmB;EAAY;GACnD,MAAM,0BAA0B,KAC/B,QAAQ;iBAmEM;EACf,MAAM;IACJ;;IAEA,OAAO;;;EAGT,WAAW;;EAEX,cAAc;EACd,cAAc;;EAEd;;EAEA;;EAEA;;;;;;;;;wBAUc,wBACd,GAAG,iBACH,GAAG,iBACH;EACE;EACA;EACA;;EAEA;IAED;;wBA0Ea,WAAW,OAAO,iBAAiB;;;YC5LvC;YACA;;iBAGK;WACN;WACA,IAAI;WACJ,OAAO,sBAAsB;;iBAGvB;;WAEN;WACA,eAAe;;WAEf;;WAEA;;WAEA;;iBAGM;WACN;WACA;;;;;;;iBAQM;;WAEN;;WAEA;;WAEA;;WAEA;;WAEA,SAAS,SAAS;;;;;WAKlB,aAAa,SAAS;;WAEtB,mBAAmB;;WAEnB,oBAAoB;;WAEpB,kBAAkB;;WAElB,mBAAmB;;;iBAIb;;WAEN;;WAEA;WACA;WACA;WACA;;WAEA,YAAY,SAAS;;WAErB;;WAEA;;WAEA;;YAGC;YACA;iBAYK;WACN;WACA;WACA;;iBAGM;WACN,IAAI;WACJ,MAAM;;WAEN;WACA;WACA,mBAAmB;;WAEnB;;;wBA2BK,UAAU,MAAM,WAAW,OAAO,KAAK;;;;;;wBAyDvC,aACd,OAAO,gBACP,OAAO,0BACN;iBAsCc;;WAEN;;;;;;;;;wBAUK,cACd,OAAO,gBACP,qBAAqB,kBACrB,OAAO,yBACP,UAAS,iBACR;;;;;;;;;;;qBAkFU;WAEL;WAEF;;WAGE;WAEF;;WAGE;WACK;;WAGL;WACK;;;wBAKG,sBAAsB,kCAAkC;;iBAqCvD;WACN;WACA;;WAEA;;;;;;;wBAQK,kBACd,iBAAiB,mBACjB;EAAW;EAA6B;IACvC;;wBAuCa,WAAW,OAAO,gBAAgB,OAAO;;;;;;;;;;;;;;;;;;qBAsB5C;YACD,yBAAyB;;wBAkBrB,2BAA2B,OAAO,gBAAgB,OAAO;EAa7D;;;EACA;;;iBAIK;WACN,SAAS;WACT;WACA;WACA;;;wBAyBK,wBAAwB,eAAe;;;;;wBA8BvC,cACd,OAAO,gBACP,QAAQ;WAAsC;UAC9C;EAAW;EAAgB;IAC1B;;;;;qBA8CU;WACX;aACE;aACA;;WAGF;aACE;aACA;;WAGF;aACE;aACA;;WAEF;aACE;aACA;;;YAKQ,0CAA0C;iBAErC;WACN;;;;;WAKA;;WAEA;WACA;;;wBAIK,8BACd,OAAO,gBACP,OAAO,aACP,SAAS,iCACR,wBAAwB;;wBA+BX,gBACd,QAAQ;WAAmC;UAC3C;EAAW;IACV;;;;;;wBAqCa,6BAA6B,OAAO,iBAAiB;;wBAmBrD,cACd,mBAAmB;WAAoC;UACvD;EAAW;IACV;qBAsCU;iBAEI;;WAEN,sBAAsB;;iBAGhB;WACN;WACA;;WAEA;WACA,kBAAkB;;WAElB;;WAEA;;;wBAIK,qBAAqB,SAAQ;;wBAS7B,sBACd,kBAAkB,mBAClB,SAAQ,uBACP;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBC5zBc;;EAEf;;EAEA;;EAEA,OAAO;;iBAGQ;EACf;EACA;EACA;;EAEA;;EAEA;;EAEA,UAAU;;iBAGK;EACf;EACA,QAAQ;;EAER;;EAEA,MAAM;;iBAGS;EACf;EACA,SAAS;;;;;EAKT,cAAc,QAAQ,uBAAuB;IAC3C;IACA;IACA;IACA,UAAU;;;wBAIQ,gBAAgB,MAAM,yBAAyB,QAAQ;iBAmB5D,sBAAsB;;EAErC;;EAEA,SAAS,sBAAsB,QAAQ;;EAEvC,UAAU,SAAS,MAAM;;iBAGV,qBAAqB;EACpC,MAAM;EACN;EACA;EACA;;EAEA;;;wBAIoB,QAAQ,GAAG,MAAM,sBAAsB,KAAK,QAAQ,qBAAqB;iBAqB9E,uBAAuB;EACtC;EACA,SAAS,sBAAsB,QAAQ;;EAEvC,YAAY,SAAS;;iBAGN,sBAAsB;;EAErC;;EAEA;;EAEA,WAAW;;EAEX,gBAAgB;;EAEhB,UAAU;;;;;;wBAOU,gBAAgB,GACpC,MAAM,uBAAuB,KAC5B,QAAQ,sBAAsB;;;;;;;iBAkChB;EACf;EACA;EACA;EACA;;wBAGc,eAAe,QAAQ,qBAAqB;;;;;;;;;;;;YC7LhD;iBAOK,qBAAqB;EACpC,MAAM;;EAEN,QAAQ,UAAU,MAAM,QAAQ,KAAK;;EAErC;;iBAGe,wBAAwB;;EAEvC,aAAa,GAAG;;EAEhB,WAAW;;;;;EAKX,YAAY;EACZ,eAAe,qBAAqB;;;;;;EAMpC,UAAU,GAAG,MAAM;;iBAGJ;;EAEf;;EAEA;;;;;EAKA;;iBAGe;EACf,aAAa;IACX;IACA;IACA;IACA;;;EAGF;IAAc;IAAW;;;EAEzB;EACA;EACA;EACA;;EAEA;;EAEA;;wBAGoB,sBAAsB,GAC1C,OAAO,wBAAwB,IAC/B,OAAM,4BACL,QAAQ;;;;;;;wBAyGK,gBAAgB;EAAY;GAC1C,uBACA,UAAS,cAAc,yBACtB,qBAAqB;;;;;;wBAoBR,aAAa;EAAY;GACvC,iBAAiB,gBAAgB,8BACjC,eACC,qBAAqB;;;;;;wBAgBR,uBAAuB;EAAY;GACjD,gBACA,iCACC,qBAAqB;;;;;;;;;;;;;UClMP;;;;;;;;;;;EAWf,OAAO;;;;YCKG;iBAKK;;EAEf;;EAEA;EACA;;EAEA;EACA;;EAEA;;;;;;EAMA;IAAW;IAAgB;;;EAE3B;;;;;EAKA;IACE;IACA;IACA;IACA;IACA;IACA;;;iBAIa;EACf,WAAW;;;;;EAKX;;;;;;EAMA,QAAQ;;EAER;;iBAGe;EACf,OAAO;;EAEP;;EAEA;;EAEA;;EAEA,UAAU;;;;;;EAMV;;;;;;;;;;;;;;;;;;;;wBAuCc,mBACd,OAAO,qBACP,OAAM,4BACL;;;;;;;;;;;;;;;;;;wBAqMmB,YACpB,SAAS,oBACT,SAAS,YACT,SAAS,qBACR,QAAQ;EAAQ;EAAgB;EAAgB;;;;;;;;;;;wBAkCnC,kBACd,SAAS,oBACT,SAAS,qBACR;EAAQ;EAAoB;EAAqB;EAAuB;;;;iBChX1D;;EAEf;EACA;;EAEA;;EAEA,MAAM;;EAEN;;EAEA;;;;;EAKA;;EAEA;;EAEA;;iBAGe;;EAEf,WAAW;;EAEX,MAAM,MAAM,OAAO,QAAQ,KAAK;;iBAGjB;;;;;;EAMf,SAAS;;EAET,aAAa,MAAM;;wBAGC,mBACpB,OAAO,YACP,eACA,MAAM,4BACL,QAAQ;iBAgCM;EACf;EACA;EACA;;EAEA;;EAEA;;EAEA;EACA;;wBAGc,yBAAyB,aAAa,eAAe;iBA+CpD;;EAEf;EACA;;EAEA;EACA;;EAEA;EACA;;EAEA;EACA;;;;;;;;;;;;;;;;;wBAkBc,iBACd,kBAAkB,YAAY,eAC9B;EAAQ;EAAoB;IAC3B;;;iBCrJc;;EAEf,WAAW,2BAA2B;;EAEtC,eAAe,2BAA2B;;iBAG3B;EACf;EACA;EACA;;EAEA;;EAEA,OAAO;;;YAIG,iBAAiB;;;;;;;;;;;;;;;;wBAwBP,UACpB,SAAS,oBACT,SAAS,YACT,SAAS,iBACR,QAAQ;;wBAwCK,WAAW,MAAM;;YAOrB,+BAA+B;iBAE1B,oBACP,KAAK;;EAEb,WAAW,2BAA2B;;EAEtC,eAAe,2BAA2B;;iBAG3B;EACf;EACA;EACA;;EAEA;EACA,OAAO;;;;;;;;;;;;;;;;;;;wBAoBa,WACpB,OAAO,qBACP,SAAS,cACR,QAAQ;wBAyDK,YAAY,MAAM;iBAMjB,mBAAmB;;;;;EAKlC,WAAW,2BAA2B;;EAEtC,eAAe,2BAA2B;;EAE1C,YAAY,MAAM;;EAElB,WAAW,MAAM;;iBAGF;;EAEf,UAAU;EACV,OAAO;;;;;;;;;;;;;;;;;;;wBAoBa,UACpB,OAAO,qBACP,SAAS,aACR,QAAQ;wBAwDK,WAAW,MAAM;iBAMhB;;EAEf,WAAW,2BAA2B;;EAEtC,aAAa,eAAe,4BAA4B;;EAExD,YAAY,eAAe,uCAAuC;;iBAGnD;EACf;;EAEA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,OAAO;;iBAGQ,uBAAuB;;;;;;;;;;EAUtC;;;;;;;;;;;;;;;;;;;;;wBAsBoB,UACpB,SAAS,qBACT,SAAS,YACT,SAAS,iBACR,QAAQ;wBA8FK,WAAW,MAAM;iBAMhB;EACf;EACA;EACA;EACA;EACA;EACA;;;;;;;;;;;;wBAoBc,mBAAmB,aAAa,cAAc,SAAS;;;YCpf3D;cACN;YACM,wBAAwB;wBAIpB,uBAAuB,iBAAiB;;;iBA2BvC;EACf;EACA;;EAEA;;;EAGA;;;EAGA;IAAU,MAAM;IAAY;IAAgB;;EAC5C;EACA;EACA;;EAEA;;EAEA,UAAU;;EAEV;IACE;IACA;IACA;;;iBAIa;EACf;EACA;EACA;EACA;EACA;EACA;;iBAGe;EACf;;EAEA;;EAEA,QAAQ,OAAO;EACf,QAAQ;;EAER;;EAEA;EACA;EACA;IAAe;IAAe;;EAC9B;;;;;;EAMA;;iBAGe,0BAA0B;EACzC,SAAS;EACT,WAAW,QAAQ,OAAO;EAC1B,OAAO;;iBAGQ;EACf,UAAU;;EAEV,OAAO;;;;;;;;;;wBA0Da,eACpB,OAAO,qBACP,SAAS,cAAc,YACvB,QAAQ,iBACR;EAAgB,SAAS;EAAoB,SAAS;IACrD,QAAQ;;wBA2DK,oBAAoB,GAAG;;;;;YC1O3B,eAAe;EAAc;EAAiB;;iBAEzC;EACf;;;EAGA;EACA;;;;;;;;wBASc,eAAe,SAAS,gBAAgB,qBAAqB;;wBAqB7D,WAAW,qBAAqB;iBAmB/B;;EAEf;;EAEA,SAAS;;EAET;;;;;;;;;;;;;;;wBAgBoB,uBACpB,oBACA,QAAQ,iBACR,OAAM,iBACL,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBCnEM;;EAEf;;EAEA;;;;;;EAMA;;;;;;EAMA;;;;;;EAMA;;;;;;;EAOA;;iBAGe;;EAEf;;EAEA;;iBAGe;;EAEf;;EAEA;;EAEA;;EAEA;;;;;EAKA;;EAEA,qBAAqB;;iBAGN;;;;;;EAMf;;EAEA;IAAe;IAAa;;;;;;;;wBAQd,4BACd,cAAc,uBACd,OAAM,mBACL;;;;;;wBA4Ca,kCACd,cAAc,uBACd,OAAM,mBACL;;;;;;;;;;;;;;;;;;;;;wBAwDa,aACd,cAAc,uBACd,OAAM,mBACL;;;;;;wBAuEa,qBACd,cAAc,uBACd,OAAM;EACH,KAAK;EAAmB,OAAO;EAAmB,IAAI;;;;iBCvQ1C;EACf,UAAU;;EAEV,eAAe;;EAEf;;EAEA;;;;;;EAMA,YAAY,QAAQ,0CAA0C;;;;;;qBAOnD,wCAAwC;mBAClC;UACT;EAER,YAAY,MAAM;EAaZ,gBAAgB,MAAM,cAAc,QAAQ;EA0C5C,cAAc,UAAU,gBAAgB,QAAQ;EA+DhD,YAAY,SAAS,kBAAkB,UAAU,iBAAiB,QAAQ;EAS1E,eAAe,MAAM,iBAAiB,QAAQ;;;;;;EAiD9C,iBAAiB,MAAM,cAAc,QAAQ;;;;;;EAiBnD,UAAU,QAAQ;EAUlB,iBAAiB;;;;iBCxMF,qBAAqB,WAAW,oBAAoB;;EAEnE,UAAU;;EAEV,cAAc;;EAEd,mBAAmB;;EAEnB,eAAe;EACf,0BAA0B;;EAE1B;IACE;IACA;IACA;MAAS;MAAa;;;;;;;;;;;;;IAYtB;;;EAGF;IACE,MAAM;IACN,OAAO;IACP,MAAM;;;;;;;;;;;iBAYO;EACf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;iBAGe;EACf,UAAU;;EAEV,cAAc;;EAEd,eAAe;IAAQ;IAAe,QAAQ;;;EAE9C,aAAa;;;;;EAKb,mBAAmB;;;;EAInB,eAAe;;EAEf,eAAe;;EAEf,oBAAoB;;EAEpB;IACE,MAAM;IACN,OAAO;IACP,MAAM;;;;;EAKR;;;;EAIA;;wBAGoB,cAAc,GAAG,MAAM,qBAAqB,KAAK,QAAQ;;;iBC5I9D;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA,WAAW;;EAEX;;;;;;;;;;;wBAYc,qBACd,UAAU,gBACV,KAAK;EAAmB;IACvB;;;;;;;;wBAyBa,8BACd,QAAQ,oBACR,KAAK;EAAmB;EAAqB;GAC7C;EAAQ;IACP;;;;;;;YCpCS,oBAAoB,QAAQ,cAAc;;;;;qBAMzC;;;;;;;;;;;;;;;;;;;;;;qBAuCA,yBAAyB;;;;;;;;;wBAmDtB,aAAa,GAAG,wBAAwB,GAAG;;;;;;;wBAqC3C,cAAc,kBAAkB;;;wBAsBhC,YAAY,eAAe;iBAU1B;;EAEf;;;EAGA;;EAEA;;iBAGe;EACf;;EAEA;;EAEA,WAAW;;EAEX;;EAEA;;YAGU;iBAEK;EACf,cAAc;;;;EAId;;;EAGA;;;EAGA,SAAS;;iBAGM;;EAEf,WAAW;;EAEX;;;;;;;;wBASc,kBACd,WAAW,aACX,YAAY,aACZ,OAAM,qBACL;iBAuHc;;;EAGf;;;EAGA;;iBAGe;EACf;EACA;;EAEA;;;EAGA;;;;;;;;;wBAUc,cACd,WAAW,aACX,YAAY,aACZ,OAAM,kBACL;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBC/Xc;;EAEf;;EAEA;;;;;EAKA;;;;;;EAMA;;iBAGe;EACf;;EAEA;;EAEA;;EAEA;;EAEA;;iBAGe;EACf,SAAS;;EAET;;EAEA;EACA;;;;;;;;;;;wBAYc,gBACd,UAAU,mBACV;EAAQ;EAAoB;EAAwB;IACnD;;;;;;;;;iBA+Gc;;EAEf;;EAEA;;wBAGc,eACd,SAAS,qBACT,SAAS,iBACT,OAAM;EACH;EAAqB;;;;;;;;iBA2BT;EACf,MAAM;IACJ;;IAEA;IACA;;;;;;EAMF;;wBAGc,0BACd,OAAO,iCACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBC/MU;;iBAKI;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;EACA,KAAK;EACL;;iBAGe;EACf;EACA;EACA;EACA;EACA;EACA;;iBAGe;EACf;EACA,OAAO;;;iBAIQ;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA,kBAAkB;IAAQ;IAAkB;;;;iBAI7B;EACf;EACA;EACA;;iBAGe;EACf;;;EAGA;;EAEA;EACA;;;iBAIe;EACf;EACA;EACA;EACA;EACA,mBAAmB;EACnB;EACA;IAAU;IAAyB;;;iBAKpB;EACf;EACA;EACA;;EAEA;EACA;;YAGU;iBAEK;EACf,SAAS;EACT;EACA;EACA;EACA;;iBAGe;EACf,eAAe;;EAEf;EACA;EACA;EACA;IACE;IACA;IACA;IACA;IACA;IACA;;EAEF;;IAEE;;IAEA;;IAEA;;IAEA;IACA;;;EAGF;IACE;MAAU;MAAe;;IACzB,OAAO;;EAET;IACE;;IAEA;IACA;IACA;IACA;IACA;IACA;IACA;;;IAGA;IACA,wBAAwB;IACxB;IACA;IACA;IACA;;EAEF,KAAK;EACL;IACE;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;;;iBAIa;EACf;EACA;;;;EAIA;EACA,KAAK,OAAO;EACZ,UAAU;IAAiB;IAAc;IAAoB;;;iBAO9C;EACf,WAAW;EACX,OAAO;EACP,OAAO;EACP;EACA;EACA;EACA;EACA;EACA;EACA;EACA,SAAS;EACT;;;;;;wBAiEc,wBAAwB,MAAM,8BAA8B;wBA8I5D,0BAA0B,MAAM,uBAAuB;wBAwBvD,wBAAwB,MAAM;iBAM7B;EACf;;EAEA;;iBAGe;EACf;;EAEA;;EAEA,SAAS,eAAe;;EAExB;EACA;;iBAGe;EACf,MAAM;EACN,SAAS;EACT;IACE;IACA;IACA;IACA;IACA,SAAS;MAAiB;MAAoB;MAAsB;;;;;;;;;wBAuExD,4BACd,QAAQ,yBACP"}
|
package/dist/rl.js
CHANGED
|
@@ -15,7 +15,7 @@ import { s as assertRewardGate } from "./schema-BuiujZ-g.js";
|
|
|
15
15
|
import { n as isTrainingLineEligible, s as toSftRows$1, t as isSplitEligible } from "./exporters-X4xwla8A.js";
|
|
16
16
|
import { t as mintRolloutRows } from "./mint-CWG-s2mI.js";
|
|
17
17
|
import { t as evaluateInterimReleaseConfidence } from "./sequential-CzK5DarL.js";
|
|
18
|
-
import {
|
|
18
|
+
import { n as prepareSystemOneReview } from "./systemone-review-DTClwgmE.js";
|
|
19
19
|
import { n as assertUniqueObservationIds, s as validateOutcomeMetricSpecifications, t as rubricPredictiveValidity } from "./rubric-predictive-validity-BsgkBfT0.js";
|
|
20
20
|
import { t as varianceBasedCurriculum } from "./active-curriculum-B_Lc9kfO.js";
|
|
21
21
|
import { n as InMemoryOutcomeStore, t as FileSystemOutcomeStore } from "./outcome-store-DUcMj-3D.js";
|
|
@@ -453,7 +453,7 @@ function checkerReadSignal(events, options) {
|
|
|
453
453
|
}]
|
|
454
454
|
};
|
|
455
455
|
}
|
|
456
|
-
/** What the referee and
|
|
456
|
+
/** What the referee and the System One model read about a claim: the source and the claim's parameters, never its prose. */
|
|
457
457
|
function claimBrief(claim, scope) {
|
|
458
458
|
const parameters = Object.fromEntries(Object.entries(claim.parameters).map(([name, value]) => [name, JSON.stringify(value).length > 400 ? `${JSON.stringify(value).slice(0, 400)}...` : value]));
|
|
459
459
|
return {
|
|
@@ -587,10 +587,10 @@ function refereeSignal(claim, answer, options = {}) {
|
|
|
587
587
|
};
|
|
588
588
|
}
|
|
589
589
|
/**
|
|
590
|
-
*
|
|
590
|
+
* System One typed questions over the same brief the referee reads. Each names a way the claim can be true of
|
|
591
591
|
* the checker and not of the statement; `true` supports that hypothesis, except `inScope`.
|
|
592
592
|
*/
|
|
593
|
-
const
|
|
593
|
+
const claimIntegritySystemOneQuestions = {
|
|
594
594
|
excludedBySource: {
|
|
595
595
|
type: "noul",
|
|
596
596
|
instructions: "Using only the source excerpts, does the source exclude the claim's case from its statement, by a hypothesis, a definition, or an assumption it states (for example a proof that calls a case easy and then assumes it away)?"
|
|
@@ -608,8 +608,8 @@ const claimIntegrityJevQuestions = {
|
|
|
608
608
|
instructions: "Is the claim's case inside the statement the source means, so that a verified result there would count for the source's authors?"
|
|
609
609
|
}
|
|
610
610
|
};
|
|
611
|
-
/** A prepared
|
|
612
|
-
function
|
|
611
|
+
/** A prepared System One review of one claim, for `systemOneEvaluator` or any transport; assess with `assessSystemOneReview`. */
|
|
612
|
+
function claimIntegritySystemOneReview(claim, scope, options) {
|
|
613
613
|
const { supportAtLeast, refuteAtLeast } = options;
|
|
614
614
|
const check = (hypothesis, supports) => ({
|
|
615
615
|
claim: hypothesis,
|
|
@@ -626,12 +626,12 @@ function claimIntegrityJevReview(claim, scope, options) {
|
|
|
626
626
|
uri: `claim:${claim.id}`
|
|
627
627
|
}]
|
|
628
628
|
});
|
|
629
|
-
return
|
|
630
|
-
version: options.version ?? `claim-integrity-
|
|
629
|
+
return prepareSystemOneReview({
|
|
630
|
+
version: options.version ?? `claim-integrity-systemone-2`,
|
|
631
631
|
request: {
|
|
632
632
|
model: options.model,
|
|
633
633
|
state: claimBrief(claim, scope),
|
|
634
|
-
questions:
|
|
634
|
+
questions: claimIntegritySystemOneQuestions
|
|
635
635
|
},
|
|
636
636
|
checks: {
|
|
637
637
|
excludedBySource: check("The source excludes the claimed case", "true"),
|
|
@@ -641,14 +641,14 @@ function claimIntegrityJevReview(claim, scope, options) {
|
|
|
641
641
|
}
|
|
642
642
|
});
|
|
643
643
|
}
|
|
644
|
-
/**
|
|
645
|
-
function
|
|
644
|
+
/** The System One signal from an assessed review: any question supported against the claim fires it. */
|
|
645
|
+
function systemOneSignal(report, options = {}) {
|
|
646
646
|
const cost = options.costUsd === void 0 ? {} : { costUsd: options.costUsd };
|
|
647
647
|
if (report === null || "error" in report) return {
|
|
648
|
-
id: "
|
|
648
|
+
id: "systemone",
|
|
649
649
|
kind: "model",
|
|
650
650
|
fired: null,
|
|
651
|
-
detail: report === null ? "
|
|
651
|
+
detail: report === null ? "System One did not run" : `System One failed: ${report.error}`,
|
|
652
652
|
evidence: [],
|
|
653
653
|
...cost
|
|
654
654
|
};
|
|
@@ -656,14 +656,14 @@ function jevSignal(report, options = {}) {
|
|
|
656
656
|
const decided = report.assessments.filter((item) => item.status !== "unresolved");
|
|
657
657
|
const evidence = report.assessments.map((item) => ({
|
|
658
658
|
kind: "model",
|
|
659
|
-
ref: `
|
|
659
|
+
ref: `systemone:${report.servedModel}:${item.question}:${item.status}`,
|
|
660
660
|
quote: `support ${item.supportProbability.toFixed(3)}, refutation ${item.refutationProbability.toFixed(3)}`
|
|
661
661
|
}));
|
|
662
662
|
return {
|
|
663
|
-
id: "
|
|
663
|
+
id: "systemone",
|
|
664
664
|
kind: "model",
|
|
665
665
|
fired: against.length > 0 ? true : decided.length > 0 ? false : null,
|
|
666
|
-
detail: against.length > 0 ? `
|
|
666
|
+
detail: against.length > 0 ? `System One supports: ${against.map((item) => item.check.claim).join("; ")}` : `System One supports no hypothesis against the claim (${decided.length} of ${report.assessments.length} questions decided)`,
|
|
667
667
|
evidence,
|
|
668
668
|
...cost
|
|
669
669
|
};
|
|
@@ -3711,6 +3711,6 @@ function loadVerifiedFindingsDataset(source) {
|
|
|
3711
3711
|
};
|
|
3712
3712
|
}
|
|
3713
3713
|
//#endregion
|
|
3714
|
-
export { ABSENT_CATEGORY, CLAIM_INTEGRITY_VERSION, FileSystemOutcomeStore, GRADER_REFERENCE_PHRASES, InMemoryOutcomeStore, PredictiveValidityResearcher, REFEREE_VERDICTS, VERIFIED_FINDING_SCHEMA, appendToCorpus, applyEloUpdate, bestOfN, bucketLabel, buildDatasetFromCorpus, buildPairwiseFromCampaign, buildRlDataset, buildVerifiedFindingRow, campaignToRunRecords, checkerReadSignal, claimBrief, claimIntegrityDigest,
|
|
3714
|
+
export { ABSENT_CATEGORY, CLAIM_INTEGRITY_VERSION, FileSystemOutcomeStore, GRADER_REFERENCE_PHRASES, InMemoryOutcomeStore, PredictiveValidityResearcher, REFEREE_VERDICTS, VERIFIED_FINDING_SCHEMA, appendToCorpus, applyEloUpdate, bestOfN, bucketLabel, buildDatasetFromCorpus, buildPairwiseFromCampaign, buildRlDataset, buildVerifiedFindingRow, campaignToRunRecords, checkerReadSignal, claimBrief, claimIntegrityDigest, claimIntegritySystemOneQuestions, claimIntegritySystemOneReview, claimIntegrityTraceQuestions, claimIntegrityVerdict, compareAdaptationCurves, datasheetToMarkdown, defaultBehaviorFeatures, detectRewardHacking, doublyRobust, easyModeCheck, extractPreferences, extractStepRewards, extractVerifiableReward, extractVerifiableRewardsFromRecords, farmingSignal, filterDeterministicallyRewarded, firstPassK, fitBradleyTerry, graderReferenceSignal, injectIrrelevantClause, inverseProbabilityWeighting, jsDivergence, loadVerifiedFindingsDataset, monitorSignal, offPolicyEstimateAll, paretoFrontier, parseSourceScopeReferee, prmTrainingPairs, quantileEdges, readCorpus, refereeSignal, renameVariables, ruleHolds, runAdaptationCurve, runComputeCurve, runContaminationProbe, runEvalCampaign, runRLCampaign, runwiseStepRewardSummary, scopeSignals, selfConsistency, selfNormalizedImportanceWeighting, shuffleOrder, simFidelityReport, sourceScopeRefereeMessages, stepRewardsToJsonl, summarizeVerifiedFindings, systemOneSignal, toAnthropicFormat, toDpoJsonl, toDpoRows, toGrpoJsonl, toGrpoRows, toPrmJsonl, toPrmRows, toSftJsonl, toSftRows, toTRLFormat, validateDatasetFormats, varianceBasedCurriculum, verificationReportToRunRecord, verifiedFindingsToJsonl };
|
|
3715
3715
|
|
|
3716
3716
|
//# sourceMappingURL=rl.js.map
|