@evomap/evolver-core 2.0.0-beta.9 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/algo/candidateAssembly.d.ts +7 -2
- package/dist/algo/candidateAssembly.js +81 -13
- package/dist/algo/capabilityCandidates.js +10 -0
- package/dist/algo/cycleEngine.d.ts +23 -2
- package/dist/algo/cycleEngine.js +236 -36
- package/dist/algo/exploration.d.ts +7 -0
- package/dist/algo/exploration.js +16 -3
- package/dist/algo/geneHealth.d.ts +2 -2
- package/dist/algo/geneHealth.js +5 -4
- package/dist/algo/geneSelection.d.ts +63 -5
- package/dist/algo/geneSelection.js +330 -30
- package/dist/algo/index.d.ts +2 -0
- package/dist/algo/index.js +2 -0
- package/dist/algo/orchestrator.d.ts +8 -1
- package/dist/algo/orchestrator.js +15 -3
- package/dist/algo/publishEligibility.d.ts +34 -0
- package/dist/algo/publishEligibility.js +52 -0
- package/dist/algo/ucb1.d.ts +53 -0
- package/dist/algo/ucb1.js +156 -0
- package/dist/assetstore/assetSidecarRecords.d.ts +7 -2
- package/dist/assetstore/assetSidecarRecords.js +238 -0
- package/dist/assetstore/assetSidecarRecovery.js +2 -2
- package/dist/assetstore/assetStoreHealth.d.ts +6 -0
- package/dist/assetstore/assetStoreHealth.js +56 -14
- package/dist/assetstore/assetStoreStorage.d.ts +1 -1
- package/dist/assetstore/assetStoreStorage.js +27 -9
- package/dist/assetstore/assetSyncLedger.d.ts +86 -0
- package/dist/assetstore/assetSyncLedger.js +683 -4
- package/dist/assetstore/index.d.ts +1 -0
- package/dist/assetstore/index.js +1 -0
- package/dist/assetstore/localAssetStoreSnapshot.d.ts +51 -0
- package/dist/assetstore/localAssetStoreSnapshot.js +329 -0
- package/dist/assetstore/localJsonl.d.ts +4 -2
- package/dist/assetstore/localJsonl.js +89 -9
- package/dist/assetstore/provenance.d.ts +80 -4
- package/dist/assetstore/provenance.js +313 -5
- package/dist/assetstore/provider.d.ts +58 -1
- package/dist/assetstore/provider.js +97 -6
- package/dist/benchmark/antiGeneBenchmark.d.ts +2 -0
- package/dist/benchmark/antiGeneBenchmark.js +4 -3
- package/dist/benchmark/antiGeneRollout.d.ts +2 -0
- package/dist/benchmark/antiGeneRollout.js +4 -3
- package/dist/benchmark/index.d.ts +2 -1
- package/dist/benchmark/index.js +2 -1
- package/dist/benchmark/selectionFlatAbstention.d.ts +152 -0
- package/dist/benchmark/selectionFlatAbstention.js +481 -0
- package/dist/bootstrap/envFingerprint.d.ts +9 -0
- package/dist/bootstrap/envFingerprint.js +5 -0
- package/dist/bootstrap/index.d.ts +2 -1
- package/dist/bootstrap/index.js +2 -1
- package/dist/bootstrap/v1EnvCompat.d.ts +111 -0
- package/dist/bootstrap/v1EnvCompat.js +280 -0
- package/dist/events/eventArchive.d.ts +2 -0
- package/dist/events/eventArchive.js +13 -3
- package/dist/events/eventSchema.d.ts +7 -7
- package/dist/events/eventStore.d.ts +2 -0
- package/dist/events/eventStore.js +5 -1
- package/dist/events/ingest.d.ts +1 -0
- package/dist/events/ingest.js +1 -0
- package/dist/events/paths.d.ts +3 -1
- package/dist/events/paths.js +4 -0
- package/dist/events/public.d.ts +2 -2
- package/dist/events/public.js +2 -2
- package/dist/events/reports.d.ts +2 -0
- package/dist/events/reports.js +4 -0
- package/dist/exec/autoExec.d.ts +43 -2
- package/dist/exec/autoExec.js +76 -9
- package/dist/exec/autonomousCycle.d.ts +22 -4
- package/dist/exec/autonomousCycle.js +64 -13
- package/dist/exec/claudeBridge.d.ts +32 -7
- package/dist/exec/claudeBridge.js +281 -29
- package/dist/exec/prompt.js +5 -1
- package/dist/exec/runnerRegistry.d.ts +68 -26
- package/dist/exec/runnerRegistry.js +307 -72
- package/dist/exec/selfPr.js +1 -7
- package/dist/feedback/envelope.d.ts +61 -0
- package/dist/feedback/envelope.js +168 -0
- package/dist/feedback/index.d.ts +1 -0
- package/dist/feedback/index.js +1 -0
- package/dist/hooks/hooks.js +1 -0
- package/dist/hub/assetCallLog.d.ts +35 -1
- package/dist/hub/assetCallLog.js +124 -1
- package/dist/hub/bindings.d.ts +8 -1
- package/dist/hub/bindings.js +17 -6
- package/dist/hub/capability.d.ts +90 -3
- package/dist/hub/fake.d.ts +2 -2
- package/dist/hub/fake.js +1 -1
- package/dist/hub/questionGenerator.d.ts +5 -1
- package/dist/hub/questionGenerator.js +8 -6
- package/dist/index.d.ts +3 -1
- package/dist/index.js +4 -1
- package/dist/mailbox/dispatch.d.ts +1 -1
- package/dist/mailbox/dispatch.js +22 -6
- package/dist/mailbox/envelope.d.ts +7 -1
- package/dist/mailbox/envelope.js +9 -2
- package/dist/mailbox/ipcServer.d.ts +10 -2
- package/dist/mailbox/ipcServer.js +163 -13
- package/dist/mailbox/store.d.ts +89 -3
- package/dist/mailbox/store.js +895 -41
- package/dist/ops/evolutionGraphProjection.d.ts +20 -0
- package/dist/ops/evolutionGraphProjection.js +315 -0
- package/dist/ops/index.d.ts +2 -1
- package/dist/ops/index.js +2 -1
- package/dist/ops/selfUpdate.d.ts +8 -0
- package/dist/ops/selfUpdate.js +24 -8
- package/dist/personality/schema.d.ts +16 -16
- package/dist/schema/evolutionGraph.d.ts +784 -0
- package/dist/schema/evolutionGraph.js +187 -0
- package/dist/schema/index.d.ts +1 -0
- package/dist/schema/index.js +1 -0
- package/dist/schema/signal.d.ts +3 -3
- package/dist/schema/signal.js +1 -1
- package/dist/signals/curriculum.d.ts +55 -0
- package/dist/signals/curriculum.js +202 -0
- package/dist/signals/cycleHistoryFromEvents.js +17 -8
- package/dist/signals/expand.d.ts +15 -1
- package/dist/signals/expand.js +169 -1
- package/dist/signals/extractor.d.ts +2 -2
- package/dist/signals/extractor.js +31 -6
- package/dist/signals/index.d.ts +3 -1
- package/dist/signals/index.js +3 -1
- package/dist/signals/metaSignals.d.ts +4 -0
- package/dist/signals/metaSignals.js +42 -0
- package/dist/signals/signalGate.js +1 -1
- package/dist/signals/taskDomain.d.ts +22 -0
- package/dist/signals/taskDomain.js +43 -0
- package/dist/strategy/constraintAblation.js +115 -369
- package/dist/strategy/constraintAblationPredicates.d.ts +31 -0
- package/dist/strategy/constraintAblationPredicates.js +339 -0
- package/dist/trace/index.d.ts +3 -1
- package/dist/trace/index.js +3 -1
- package/dist/trace/learningTrace.d.ts +209 -0
- package/dist/trace/learningTrace.js +276 -0
- package/dist/trace/proxyTurns.d.ts +31 -0
- package/dist/trace/proxyTurns.js +137 -0
- package/dist/verify/sandboxRunner.d.ts +28 -0
- package/dist/verify/sandboxRunner.js +218 -19
- package/dist/verify/sandboxedValidation.d.ts +9 -0
- package/dist/verify/sandboxedValidation.js +113 -13
- package/dist/verify/validation.d.ts +11 -1
- package/dist/verify/validation.js +31 -0
- package/package.json +5 -2
|
@@ -1,4 +1,87 @@
|
|
|
1
|
-
import { computeAssetId, verifyAssetId } from '../wire/index.js';
|
|
1
|
+
import { canonicalize, computeAssetId, verifyAssetId } from '../wire/index.js';
|
|
2
|
+
export class FrozenAssetIdCollisionError extends Error {
|
|
3
|
+
assetId;
|
|
4
|
+
code = 'FROZEN_ASSET_ID_COLLISION';
|
|
5
|
+
constructor(assetId) {
|
|
6
|
+
super('frozen asset_id already exists with different content');
|
|
7
|
+
this.assetId = assetId;
|
|
8
|
+
this.name = 'FrozenAssetIdCollisionError';
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
export class InvalidFrozenPutResultError extends Error {
|
|
12
|
+
code = 'INVALID_FROZEN_PUT_RESULT';
|
|
13
|
+
constructor() {
|
|
14
|
+
super('invalid frozen put result');
|
|
15
|
+
this.name = 'InvalidFrozenPutResultError';
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
export function frozenAssetRecordsEqual(left, right) {
|
|
19
|
+
return canonicalize(left) === canonicalize(right);
|
|
20
|
+
}
|
|
21
|
+
export function validateFrozenPutResult(value, expectedAssetId) {
|
|
22
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
23
|
+
throw new InvalidFrozenPutResultError();
|
|
24
|
+
const result = value;
|
|
25
|
+
if (result['asset_id'] !== expectedAssetId
|
|
26
|
+
|| typeof result['stored'] !== 'boolean'
|
|
27
|
+
|| result['verified'] !== false) {
|
|
28
|
+
throw new InvalidFrozenPutResultError();
|
|
29
|
+
}
|
|
30
|
+
return value;
|
|
31
|
+
}
|
|
32
|
+
export class InvalidConditionalPutResultError extends Error {
|
|
33
|
+
reason;
|
|
34
|
+
code = 'INVALID_CONDITIONAL_PUT_RESULT';
|
|
35
|
+
constructor(reason) {
|
|
36
|
+
super(`invalid conditional put result: ${reason}`);
|
|
37
|
+
this.reason = reason;
|
|
38
|
+
this.name = 'InvalidConditionalPutResultError';
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
/** Validate an injected provider response before callers treat it as an explicit write/no-write decision. */
|
|
42
|
+
export function validateConditionalPutResult(value, expectedAssetId, options) {
|
|
43
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
|
44
|
+
throw new InvalidConditionalPutResultError('malformed_result');
|
|
45
|
+
}
|
|
46
|
+
const result = value;
|
|
47
|
+
const status = result['status'];
|
|
48
|
+
const collisionWithAssetId = result['collisionWithAssetId'];
|
|
49
|
+
if (typeof result['asset_id'] !== 'string'
|
|
50
|
+
|| typeof result['stored'] !== 'boolean'
|
|
51
|
+
|| typeof result['verified'] !== 'boolean'
|
|
52
|
+
|| (status !== 'stored' && status !== 'already_exists' && status !== 'logical_collision')
|
|
53
|
+
|| (result['logicalId'] !== undefined
|
|
54
|
+
&& (typeof result['logicalId'] !== 'string'
|
|
55
|
+
|| !result['logicalId'].trim()
|
|
56
|
+
|| result['logicalId'] !== result['logicalId'].trim()))
|
|
57
|
+
|| (collisionWithAssetId !== undefined
|
|
58
|
+
&& (typeof collisionWithAssetId !== 'string'
|
|
59
|
+
|| !collisionWithAssetId.trim()
|
|
60
|
+
|| collisionWithAssetId !== collisionWithAssetId.trim()))) {
|
|
61
|
+
throw new InvalidConditionalPutResultError('malformed_result');
|
|
62
|
+
}
|
|
63
|
+
if (result['asset_id'] !== expectedAssetId) {
|
|
64
|
+
throw new InvalidConditionalPutResultError('asset_id_mismatch');
|
|
65
|
+
}
|
|
66
|
+
if (result['stored'] !== (status === 'stored')) {
|
|
67
|
+
throw new InvalidConditionalPutResultError('inconsistent_status');
|
|
68
|
+
}
|
|
69
|
+
const allowLogicalCollision = options?.allowLogicalCollision === true;
|
|
70
|
+
if (status === 'logical_collision'
|
|
71
|
+
&& (!collisionWithAssetId || collisionWithAssetId === expectedAssetId || allowLogicalCollision)) {
|
|
72
|
+
throw new InvalidConditionalPutResultError('invalid_collision');
|
|
73
|
+
}
|
|
74
|
+
if (status === 'stored' && collisionWithAssetId && !allowLogicalCollision) {
|
|
75
|
+
throw new InvalidConditionalPutResultError('collision_bypass');
|
|
76
|
+
}
|
|
77
|
+
return value;
|
|
78
|
+
}
|
|
79
|
+
export function supportsAtomicConditionalPut(provider) {
|
|
80
|
+
return typeof provider.putConditional === 'function';
|
|
81
|
+
}
|
|
82
|
+
export function supportsAtomicFrozenConditionalPut(provider) {
|
|
83
|
+
return typeof provider.putFrozenConditional === 'function';
|
|
84
|
+
}
|
|
2
85
|
export class AssetIdMismatchError extends Error {
|
|
3
86
|
claimed;
|
|
4
87
|
actual;
|
|
@@ -12,6 +95,18 @@ export class AssetIdMismatchError extends Error {
|
|
|
12
95
|
export class CapsuleGeneBindingError extends Error {
|
|
13
96
|
constructor() { super('Capsule.gene 必须非空或显式 "ad-hoc" 哨兵 (批注#28/M3-4)'); this.name = 'CapsuleGeneBindingError'; }
|
|
14
97
|
}
|
|
98
|
+
/**
|
|
99
|
+
* M3-4 强绑定校验: Capsule.gene 必须非空(否则一条无来源基因的经验会污染选择池).
|
|
100
|
+
* 抽成独立 helper 以便 normalizeForPut(校验落库路径)与 ingestUnverified(冻结落库路径,
|
|
101
|
+
* 绕过 normalizeForPut)共用同一条不变量,不让降级路径把绑定校验漏掉.
|
|
102
|
+
*/
|
|
103
|
+
export function assertCapsuleGeneBinding(asset) {
|
|
104
|
+
if (asset.type !== 'Capsule')
|
|
105
|
+
return;
|
|
106
|
+
const gene = asset.gene;
|
|
107
|
+
if (typeof gene !== 'string' || gene.length === 0)
|
|
108
|
+
throw new CapsuleGeneBindingError();
|
|
109
|
+
}
|
|
15
110
|
/**
|
|
16
111
|
* 落库前规范化(共享给各 provider): 计算/校验 asset_id + 强绑定校验.
|
|
17
112
|
* - 缺 asset_id → 计算填入(verified=false 表示非入参自带).
|
|
@@ -19,11 +114,7 @@ export class CapsuleGeneBindingError extends Error {
|
|
|
19
114
|
* - Capsule.gene 必须非空(M3-4 强绑定).
|
|
20
115
|
*/
|
|
21
116
|
export function normalizeForPut(asset) {
|
|
22
|
-
|
|
23
|
-
const gene = asset.gene;
|
|
24
|
-
if (typeof gene !== 'string' || gene.length === 0)
|
|
25
|
-
throw new CapsuleGeneBindingError();
|
|
26
|
-
}
|
|
117
|
+
assertCapsuleGeneBinding(asset);
|
|
27
118
|
const actual = computeAssetId(asset);
|
|
28
119
|
if (actual === null)
|
|
29
120
|
throw new Error('computeAssetId 失败: 资产非对象');
|
|
@@ -70,6 +70,8 @@ export interface AntiGeneBenchmarkOptions {
|
|
|
70
70
|
minFailureDelta?: number;
|
|
71
71
|
now?: () => number;
|
|
72
72
|
eventsPath?: string;
|
|
73
|
+
/** Apply the pre-IDF semantic scorer consistently to both A/B arms. */
|
|
74
|
+
disableSemanticIdf?: boolean;
|
|
73
75
|
}
|
|
74
76
|
export interface AntiGeneBenchmarkDeps {
|
|
75
77
|
store: AssetStoreProvider;
|
|
@@ -79,7 +79,7 @@ function executeFor(outcome) {
|
|
|
79
79
|
},
|
|
80
80
|
});
|
|
81
81
|
}
|
|
82
|
-
async function runOneArm(arm, task, source, sourceReview, rootDir, now) {
|
|
82
|
+
async function runOneArm(arm, task, source, sourceReview, rootDir, now, disableSemanticIdf) {
|
|
83
83
|
const armDir = join(rootDir, arm, task.id);
|
|
84
84
|
const store = new LocalJsonlProvider(join(armDir, 'assets'));
|
|
85
85
|
const review = new ReviewLedger(join(armDir, 'assets'), now);
|
|
@@ -105,6 +105,7 @@ async function runOneArm(arm, task, source, sourceReview, rootDir, now) {
|
|
|
105
105
|
execute: executeFor(spec),
|
|
106
106
|
review,
|
|
107
107
|
consumePendingSignals: false,
|
|
108
|
+
...(disableSemanticIdf ? { disableSemanticIdf: true } : {}),
|
|
108
109
|
};
|
|
109
110
|
const result = await runEvolutionCycle(engine, store, opts);
|
|
110
111
|
const observed = observedWarningIds(result.decision);
|
|
@@ -168,8 +169,8 @@ export async function runAntiGeneBenchmark(suite, deps, options = {}) {
|
|
|
168
169
|
for (const task of suite.tasks) {
|
|
169
170
|
taskResults.push({
|
|
170
171
|
taskId: task.id,
|
|
171
|
-
baseline: await runOneArm('baseline', task, deps.store, sourceReview, rootDir, now),
|
|
172
|
-
antiGene: await runOneArm('antiGene', task, deps.store, sourceReview, rootDir, now),
|
|
172
|
+
baseline: await runOneArm('baseline', task, deps.store, sourceReview, rootDir, now, options.disableSemanticIdf === true),
|
|
173
|
+
antiGene: await runOneArm('antiGene', task, deps.store, sourceReview, rootDir, now, options.disableSemanticIdf === true),
|
|
173
174
|
});
|
|
174
175
|
}
|
|
175
176
|
const report = buildReport(suite, taskResults, minSamples, minFailureDelta);
|
|
@@ -75,6 +75,8 @@ export interface AntiGeneRolloutOptions {
|
|
|
75
75
|
minFailureDelta?: number;
|
|
76
76
|
now?: () => number;
|
|
77
77
|
eventsPath?: string;
|
|
78
|
+
/** Apply the pre-IDF semantic scorer consistently to both A/B arms. */
|
|
79
|
+
disableSemanticIdf?: boolean;
|
|
78
80
|
}
|
|
79
81
|
export declare function runAntiGeneRollout(suite: AntiGeneRolloutSuite, deps: AntiGeneRolloutDeps, options?: AntiGeneRolloutOptions): Promise<AntiGeneRolloutReport>;
|
|
80
82
|
export declare function writeAntiGeneRolloutResult(report: AntiGeneRolloutReport, eventsPath: string, now?: () => number): Promise<void>;
|
|
@@ -82,7 +82,7 @@ function scoreFromResult(result, status) {
|
|
|
82
82
|
const score = result.capsule?.outcome?.score;
|
|
83
83
|
return typeof score === 'number' && Number.isFinite(score) ? score : status === 'success' ? 0.9 : 0.2;
|
|
84
84
|
}
|
|
85
|
-
async function runOneArm(arm, task, source, sourceReview, makeExecute, rootDir, now) {
|
|
85
|
+
async function runOneArm(arm, task, source, sourceReview, makeExecute, rootDir, now, disableSemanticIdf) {
|
|
86
86
|
const armDir = join(rootDir, arm, task.id);
|
|
87
87
|
const store = new LocalJsonlProvider(join(armDir, 'assets'));
|
|
88
88
|
const review = new ReviewLedger(join(armDir, 'assets'), now);
|
|
@@ -107,6 +107,7 @@ async function runOneArm(arm, task, source, sourceReview, makeExecute, rootDir,
|
|
|
107
107
|
execute: makeExecute({ arm, task, store, review, armDir }),
|
|
108
108
|
review,
|
|
109
109
|
consumePendingSignals: false,
|
|
110
|
+
...(disableSemanticIdf ? { disableSemanticIdf: true } : {}),
|
|
110
111
|
};
|
|
111
112
|
const result = await runEvolutionCycle(engine, store, opts);
|
|
112
113
|
const status = statusFromResult(result);
|
|
@@ -183,8 +184,8 @@ export async function runAntiGeneRollout(suite, deps, options = {}) {
|
|
|
183
184
|
for (const task of suite.tasks) {
|
|
184
185
|
taskResults.push({
|
|
185
186
|
taskId: task.id,
|
|
186
|
-
baseline: await runOneArm('baseline', task, deps.store, sourceReview, deps.makeExecute, rootDir, now),
|
|
187
|
-
antiGene: await runOneArm('antiGene', task, deps.store, sourceReview, deps.makeExecute, rootDir, now),
|
|
187
|
+
baseline: await runOneArm('baseline', task, deps.store, sourceReview, deps.makeExecute, rootDir, now, options.disableSemanticIdf === true),
|
|
188
|
+
antiGene: await runOneArm('antiGene', task, deps.store, sourceReview, deps.makeExecute, rootDir, now, options.disableSemanticIdf === true),
|
|
188
189
|
});
|
|
189
190
|
}
|
|
190
191
|
const report = buildReport(suite, taskResults, minSamples, minFailureDelta);
|
|
@@ -3,4 +3,5 @@ export * from './thesis.js';
|
|
|
3
3
|
export * from './evolutionThesisSolver.js';
|
|
4
4
|
export * from './antiGeneBenchmark.js';
|
|
5
5
|
export * from './antiGeneRollout.js';
|
|
6
|
-
export * from './triggerShift.js';
|
|
6
|
+
export * from './triggerShift.js';
|
|
7
|
+
export * from './selectionFlatAbstention.js';
|
package/dist/benchmark/index.js
CHANGED
|
@@ -3,4 +3,5 @@ export * from './thesis.js';
|
|
|
3
3
|
export * from './evolutionThesisSolver.js';
|
|
4
4
|
export * from './antiGeneBenchmark.js';
|
|
5
5
|
export * from './antiGeneRollout.js';
|
|
6
|
-
export * from './triggerShift.js';
|
|
6
|
+
export * from './triggerShift.js';
|
|
7
|
+
export * from './selectionFlatAbstention.js';
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import { type GeneDecision, type SelectionInput } from '../algo/geneSelection.js';
|
|
2
|
+
export declare const SELECTION_FLAT_ABSTENTION_BENCHMARK_VERSION = "global-flat-abstention-paired-v1";
|
|
3
|
+
export type SelectionFlatArm = 'current' | 'global-flat-abstain';
|
|
4
|
+
export type SelectionFlatSplit = 'calibration' | 'holdout';
|
|
5
|
+
export type SelectionFlatAction = {
|
|
6
|
+
kind: 'reuse';
|
|
7
|
+
geneId: string;
|
|
8
|
+
assetId?: string;
|
|
9
|
+
} | {
|
|
10
|
+
kind: 'innovate';
|
|
11
|
+
};
|
|
12
|
+
export interface SelectionFlatComparison {
|
|
13
|
+
targeted: boolean;
|
|
14
|
+
targetReason?: 'outside_plateau_flat_positive';
|
|
15
|
+
currentDecision: GeneDecision;
|
|
16
|
+
currentAction: SelectionFlatAction;
|
|
17
|
+
globalFlatAbstainAction: SelectionFlatAction;
|
|
18
|
+
}
|
|
19
|
+
export interface SelectionFlatCohort {
|
|
20
|
+
id: string;
|
|
21
|
+
version: string;
|
|
22
|
+
sourceCommit: string;
|
|
23
|
+
taskFingerprints: readonly string[];
|
|
24
|
+
verifier: {
|
|
25
|
+
id: string;
|
|
26
|
+
version: string;
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
export type SelectionFlatExpectedRole = 'target' | 'control';
|
|
30
|
+
export interface SelectionFlatExpectation {
|
|
31
|
+
role: SelectionFlatExpectedRole;
|
|
32
|
+
currentAction: SelectionFlatAction;
|
|
33
|
+
globalFlatAbstainAction: SelectionFlatAction;
|
|
34
|
+
}
|
|
35
|
+
export interface SelectionFlatTaskSnapshot<I> {
|
|
36
|
+
id: string;
|
|
37
|
+
workspaceSnapshotDigest: string;
|
|
38
|
+
split: SelectionFlatSplit;
|
|
39
|
+
stratum: string;
|
|
40
|
+
/** Verifier input is a private structured-clone-normalized copy (for example, Node Buffer becomes Uint8Array). */
|
|
41
|
+
payload: I;
|
|
42
|
+
selection: Omit<SelectionInput, 'selectionGuard'>;
|
|
43
|
+
expectation: SelectionFlatExpectation;
|
|
44
|
+
}
|
|
45
|
+
export interface SelectionFlatTask<I> extends SelectionFlatTaskSnapshot<I> {
|
|
46
|
+
taskFingerprint: string;
|
|
47
|
+
}
|
|
48
|
+
export interface SelectionFlatBenchmarkSuite<I> {
|
|
49
|
+
name: string;
|
|
50
|
+
cohort: SelectionFlatCohort;
|
|
51
|
+
tasks: readonly SelectionFlatTask<I>[];
|
|
52
|
+
}
|
|
53
|
+
export interface SelectionFlatVerifierProof {
|
|
54
|
+
id: string;
|
|
55
|
+
version: string;
|
|
56
|
+
proofSha256: string;
|
|
57
|
+
}
|
|
58
|
+
export interface SelectionFlatObjectiveOutcome {
|
|
59
|
+
passed: boolean;
|
|
60
|
+
producedValue: boolean;
|
|
61
|
+
unsafe: boolean;
|
|
62
|
+
cost: number;
|
|
63
|
+
verifier: SelectionFlatVerifierProof;
|
|
64
|
+
}
|
|
65
|
+
export interface SelectionFlatVerificationRequest<I> {
|
|
66
|
+
task: SelectionFlatTask<I>;
|
|
67
|
+
arm: SelectionFlatArm;
|
|
68
|
+
action: SelectionFlatAction;
|
|
69
|
+
isolationKey: string;
|
|
70
|
+
order: 0 | 1;
|
|
71
|
+
}
|
|
72
|
+
export type SelectionFlatVerifier<I> = (request: SelectionFlatVerificationRequest<I>) => Promise<SelectionFlatObjectiveOutcome> | SelectionFlatObjectiveOutcome;
|
|
73
|
+
export interface SelectionFlatBenchmarkRow {
|
|
74
|
+
taskFingerprint: string;
|
|
75
|
+
split: SelectionFlatSplit;
|
|
76
|
+
stratum: string;
|
|
77
|
+
armOrder: readonly [SelectionFlatArm, SelectionFlatArm];
|
|
78
|
+
current: {
|
|
79
|
+
action: SelectionFlatAction['kind'];
|
|
80
|
+
outcome: SelectionFlatObjectiveOutcome;
|
|
81
|
+
};
|
|
82
|
+
globalFlatAbstain: {
|
|
83
|
+
action: SelectionFlatAction['kind'];
|
|
84
|
+
outcome: SelectionFlatObjectiveOutcome;
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
export interface SelectionFlatPairedStats {
|
|
88
|
+
n: number;
|
|
89
|
+
currentPasses: number;
|
|
90
|
+
globalFlatAbstainPasses: number;
|
|
91
|
+
bothPass: number;
|
|
92
|
+
bothFail: number;
|
|
93
|
+
globalOnlyPass: number;
|
|
94
|
+
currentOnlyPass: number;
|
|
95
|
+
currentPassRate: number;
|
|
96
|
+
globalFlatAbstainPassRate: number;
|
|
97
|
+
pairedRiskDifference: number;
|
|
98
|
+
ci95Low: number;
|
|
99
|
+
ci95High: number;
|
|
100
|
+
mcnemarPValue: number;
|
|
101
|
+
currentProducedValueRate: number;
|
|
102
|
+
globalFlatAbstainProducedValueRate: number;
|
|
103
|
+
currentUnsafeRate: number;
|
|
104
|
+
globalFlatAbstainUnsafeRate: number;
|
|
105
|
+
unsafeRateDelta: number;
|
|
106
|
+
currentAverageCost: number;
|
|
107
|
+
globalFlatAbstainAverageCost: number;
|
|
108
|
+
averageCostDelta: number;
|
|
109
|
+
currentInnovateRate: number;
|
|
110
|
+
globalFlatAbstainInnovateRate: number;
|
|
111
|
+
}
|
|
112
|
+
export interface SelectionFlatBenchmarkReport {
|
|
113
|
+
benchmarkVersion: typeof SELECTION_FLAT_ABSTENTION_BENCHMARK_VERSION;
|
|
114
|
+
statisticalMethods: {
|
|
115
|
+
mcnemar: 'exact-two-sided-binomial';
|
|
116
|
+
riskDifferenceCi: 'paired-hoeffding-95';
|
|
117
|
+
};
|
|
118
|
+
suite: string;
|
|
119
|
+
cohort: Omit<SelectionFlatCohort, 'taskFingerprints'> & {
|
|
120
|
+
registeredTasks: number;
|
|
121
|
+
};
|
|
122
|
+
datasetDigest: string;
|
|
123
|
+
targetTasks: number;
|
|
124
|
+
controlTasks: number;
|
|
125
|
+
controlDivergences: 0;
|
|
126
|
+
rows: SelectionFlatBenchmarkRow[];
|
|
127
|
+
overall: SelectionFlatPairedStats;
|
|
128
|
+
calibration: SelectionFlatPairedStats;
|
|
129
|
+
holdout: SelectionFlatPairedStats;
|
|
130
|
+
strata: Record<string, SelectionFlatPairedStats>;
|
|
131
|
+
}
|
|
132
|
+
export interface SelectionFlatPassPair {
|
|
133
|
+
currentPassed: boolean;
|
|
134
|
+
globalFlatAbstainPassed: boolean;
|
|
135
|
+
}
|
|
136
|
+
/** Compare production with a benchmark-only global flat-positive abstention counterfactual. */
|
|
137
|
+
export declare function compareOutsidePlateauFlatSelection(input: Omit<SelectionInput, 'selectionGuard'>): SelectionFlatComparison;
|
|
138
|
+
/** Compute the digest that the cohort manifest must pre-register for this exact task snapshot. */
|
|
139
|
+
export declare function computeSelectionFlatTaskFingerprint<I>(task: SelectionFlatTaskSnapshot<I>): string;
|
|
140
|
+
/** Exact two-sided McNemar p-value over the discordant pairs. */
|
|
141
|
+
export declare function exactMcNemarTwoSided(globalOnlyPass: number, currentOnlyPass: number): number;
|
|
142
|
+
/** Distribution-free Hoeffding 95% CI for the mean paired success difference (global minus current). */
|
|
143
|
+
export declare function pairedRiskDifference95(pairs: readonly SelectionFlatPassPair[]): {
|
|
144
|
+
difference: number;
|
|
145
|
+
ci95Low: number;
|
|
146
|
+
ci95High: number;
|
|
147
|
+
};
|
|
148
|
+
/**
|
|
149
|
+
* Run a pre-registered, full-information paired benchmark. Controls are classification checks only and are never
|
|
150
|
+
* executed. Target tasks run both arms on separately cloned inputs and caller-owned isolated workspaces.
|
|
151
|
+
*/
|
|
152
|
+
export declare function runSelectionFlatAbstentionBenchmark<I>(suite: SelectionFlatBenchmarkSuite<I>, verifier: SelectionFlatVerifier<I>): Promise<SelectionFlatBenchmarkReport>;
|