@remnic/bench 9.6.20 → 9.6.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -1
- package/dist/index.d.ts +148 -2
- package/dist/index.js +1137 -476
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -91,12 +91,20 @@ status` to report ChatGPT authentication. A 473-credit safety reserve leaves
|
|
|
91
91
|
then measure a quick task before choosing a workload bound:
|
|
92
92
|
|
|
93
93
|
```bash
|
|
94
|
+
BUILD_WEEK_RUN_ROOT="$HOME/.remnic/bench/build-week-2026"
|
|
95
|
+
export BUILD_WEEK_RESULTS_DIR="$BUILD_WEEK_RUN_ROOT/results"
|
|
96
|
+
umask 077
|
|
97
|
+
mkdir -p "$BUILD_WEEK_RUN_ROOT" "$BUILD_WEEK_RESULTS_DIR"
|
|
98
|
+
chmod 700 "$BUILD_WEEK_RUN_ROOT" "$BUILD_WEEK_RESULTS_DIR"
|
|
99
|
+
|
|
94
100
|
export REMNIC_BENCH_CODEX_CREDIT_BUDGET=2473
|
|
95
101
|
export REMNIC_BENCH_CODEX_CREDIT_RESERVE=473
|
|
96
|
-
export REMNIC_BENCH_CODEX_CREDIT_LEDGER="$
|
|
102
|
+
export REMNIC_BENCH_CODEX_CREDIT_LEDGER="$BUILD_WEEK_RUN_ROOT/codex-credit-ledger.json"
|
|
97
103
|
|
|
98
104
|
remnic bench run --quick longmemeval \
|
|
99
105
|
--runtime-profile real \
|
|
106
|
+
--results-dir "$BUILD_WEEK_RESULTS_DIR" \
|
|
107
|
+
--drain-timeout 600000 \
|
|
100
108
|
--system-provider codex-cli --system-model gpt-5.6-luna \
|
|
101
109
|
--system-codex-reasoning-effort medium \
|
|
102
110
|
--internal-provider codex-cli --internal-model gpt-5.6-luna \
|
|
@@ -106,6 +114,8 @@ remnic bench run --quick longmemeval \
|
|
|
106
114
|
|
|
107
115
|
remnic bench run longmemeval \
|
|
108
116
|
--runtime-profile real --limit <LEDGER_DERIVED_LIMIT> \
|
|
117
|
+
--results-dir "$BUILD_WEEK_RESULTS_DIR" \
|
|
118
|
+
--drain-timeout 600000 \
|
|
109
119
|
--system-provider codex-cli --system-model gpt-5.6-luna \
|
|
110
120
|
--system-codex-reasoning-effort medium \
|
|
111
121
|
--internal-provider codex-cli --internal-model gpt-5.6-luna \
|
|
@@ -124,6 +134,20 @@ Rates per one million tokens are Luna: 25 input, 2.5 cached input, 150 output;
|
|
|
124
134
|
Terra: 62.5 input, 6.25 cached input, 375 output. A bounded result is a trial,
|
|
125
135
|
not a full leaderboard artifact.
|
|
126
136
|
|
|
137
|
+
Codex CLI receives a benchmark-owned 180-second transport timeout when no
|
|
138
|
+
request timeout is supplied. Keep `--request-timeout` out of these commands:
|
|
139
|
+
an explicit value also becomes a whole-phase guard, while the transport-only
|
|
140
|
+
default lets long store/recall/reset phases complete. The 600-second drain cap
|
|
141
|
+
remains explicit for queued internal work.
|
|
142
|
+
|
|
143
|
+
The ledger and results stay outside the repository because stored runs may
|
|
144
|
+
contain questions, answers, and recalled context. The `umask` plus explicit
|
|
145
|
+
directory modes make newly created state private. After the first ledger write,
|
|
146
|
+
run `chmod 600 "$REMNIC_BENCH_CODEX_CREDIT_LEDGER"`. Preserve the exact run ID
|
|
147
|
+
printed by the CLI, or recover it only from this run store with
|
|
148
|
+
`remnic bench runs list --results-dir "$BUILD_WEEK_RESULTS_DIR"`; use that ID
|
|
149
|
+
for export and artifact promotion rather than an ambiguous “latest” run.
|
|
150
|
+
|
|
127
151
|
Codex built and adversarially reviewed the Build Week adapter, Responses
|
|
128
152
|
provider, and report card. The underlying Remnic engine and original benchmark
|
|
129
153
|
harness are prior work. The evidence ledger, credit-backed frontier-run
|
package/dist/index.d.ts
CHANGED
|
@@ -300,6 +300,12 @@ interface ProviderConfig {
|
|
|
300
300
|
retryOnTimeout?: boolean;
|
|
301
301
|
max429WaitMs?: number;
|
|
302
302
|
};
|
|
303
|
+
/**
|
|
304
|
+
* Provider transport timeout that must not be interpreted as a benchmark
|
|
305
|
+
* phase timeout. Runtime profiles use this for safe provider defaults while
|
|
306
|
+
* reserving retryOptions.timeoutMs for an explicit --request-timeout.
|
|
307
|
+
*/
|
|
308
|
+
providerRequestTimeoutMs?: number;
|
|
303
309
|
disableThinking?: boolean;
|
|
304
310
|
reasoningEffort?: BenchReasoningEffort;
|
|
305
311
|
responderContextBudgetChars?: number;
|
|
@@ -1506,6 +1512,36 @@ declare const BENCHMARK_RESULT_SCHEMA: {
|
|
|
1506
1512
|
};
|
|
1507
1513
|
};
|
|
1508
1514
|
|
|
1515
|
+
interface CodexCliNativeUsage {
|
|
1516
|
+
inputTokens: number;
|
|
1517
|
+
cachedInputTokens: number;
|
|
1518
|
+
outputTokens: number;
|
|
1519
|
+
reasoningOutputTokens: number;
|
|
1520
|
+
}
|
|
1521
|
+
interface CodexCreditReceiptScope extends CodexCliNativeUsage {
|
|
1522
|
+
calls: number;
|
|
1523
|
+
credits: number;
|
|
1524
|
+
models: Array<CodexCliNativeUsage & {
|
|
1525
|
+
model: string;
|
|
1526
|
+
calls: number;
|
|
1527
|
+
credits: number;
|
|
1528
|
+
}>;
|
|
1529
|
+
}
|
|
1530
|
+
interface CodexCreditReceipt {
|
|
1531
|
+
schemaVersion: 1;
|
|
1532
|
+
ledgerSha256: string;
|
|
1533
|
+
budgetCredits: number;
|
|
1534
|
+
reserveCredits: number;
|
|
1535
|
+
plannedSpendCeilingCredits: number;
|
|
1536
|
+
totalSpentCredits: number;
|
|
1537
|
+
totalRemainingCredits: number;
|
|
1538
|
+
blocked: boolean;
|
|
1539
|
+
cumulative: CodexCreditReceiptScope;
|
|
1540
|
+
run?: CodexCreditReceiptScope & {
|
|
1541
|
+
id: string;
|
|
1542
|
+
};
|
|
1543
|
+
}
|
|
1544
|
+
|
|
1509
1545
|
declare const BENCHMARK_REPRO_MANIFEST_FILENAME = "MANIFEST.json";
|
|
1510
1546
|
declare const BENCHMARK_REPRO_MANIFEST_SCHEMA_VERSION = 1;
|
|
1511
1547
|
interface BenchmarkReproManifestFile {
|
|
@@ -1591,6 +1627,7 @@ interface BenchmarkReproManifest {
|
|
|
1591
1627
|
}>;
|
|
1592
1628
|
datasets: BenchmarkReproManifestDataset[];
|
|
1593
1629
|
results: BenchmarkReproManifestResult[];
|
|
1630
|
+
codexCredit?: CodexCreditReceipt;
|
|
1594
1631
|
artifactHash: string;
|
|
1595
1632
|
}
|
|
1596
1633
|
interface BuildBenchmarkReproManifestOptions {
|
|
@@ -3023,6 +3060,115 @@ interface DiagnoseLoComoProfileDeltaOptions {
|
|
|
3023
3060
|
declare function diagnoseLoComoProfileDelta(options: DiagnoseLoComoProfileDeltaOptions): LoComoProfileDeltaReport;
|
|
3024
3061
|
declare function renderLoComoProfileDeltaMarkdown(report: LoComoProfileDeltaReport): string;
|
|
3025
3062
|
|
|
3063
|
+
declare const LOCOMO_FULL_TASK_COUNT = 1986;
|
|
3064
|
+
declare const LOCOMO_RECALL_EXCERPT_CHARS = 240;
|
|
3065
|
+
declare const LOCOMO_RECALL_DIFF_LINE_LIMIT = 20;
|
|
3066
|
+
interface LoComoRawResultEvidence {
|
|
3067
|
+
result: BenchmarkResult;
|
|
3068
|
+
reference: string;
|
|
3069
|
+
/** SHA-256 of the exact result-file bytes, computed before JSON parsing. */
|
|
3070
|
+
sha256: string;
|
|
3071
|
+
}
|
|
3072
|
+
interface LoComoRecallTextDigest {
|
|
3073
|
+
sha256: string;
|
|
3074
|
+
charCount: number;
|
|
3075
|
+
excerpt: string;
|
|
3076
|
+
}
|
|
3077
|
+
interface LoComoRecallLineEvidence extends LoComoRecallTextDigest {
|
|
3078
|
+
ordinal: number;
|
|
3079
|
+
sourceRef?: string;
|
|
3080
|
+
}
|
|
3081
|
+
interface LoComoRecallLineDelta {
|
|
3082
|
+
totalCount: number;
|
|
3083
|
+
shownCount: number;
|
|
3084
|
+
lines: LoComoRecallLineEvidence[];
|
|
3085
|
+
}
|
|
3086
|
+
interface LoComoRecallContextSummary {
|
|
3087
|
+
sha256: string;
|
|
3088
|
+
charCount: number;
|
|
3089
|
+
lineCount: number;
|
|
3090
|
+
headings: string[];
|
|
3091
|
+
sourceRefs: string[];
|
|
3092
|
+
expectedTokenCoverage: number;
|
|
3093
|
+
}
|
|
3094
|
+
interface LoComoRecallMetricDelta {
|
|
3095
|
+
baselineMean: number;
|
|
3096
|
+
realMean: number;
|
|
3097
|
+
delta: number;
|
|
3098
|
+
wins: number;
|
|
3099
|
+
losses: number;
|
|
3100
|
+
ties: number;
|
|
3101
|
+
}
|
|
3102
|
+
interface LoComoRecallCategoryDelta extends LoComoRecallMetricDelta {
|
|
3103
|
+
category: string;
|
|
3104
|
+
taskCount: number;
|
|
3105
|
+
}
|
|
3106
|
+
interface LoComoFinalContextRegression {
|
|
3107
|
+
taskId: string;
|
|
3108
|
+
category: string;
|
|
3109
|
+
baselineScore: number;
|
|
3110
|
+
realScore: number;
|
|
3111
|
+
delta: number;
|
|
3112
|
+
questionSha256: string;
|
|
3113
|
+
expectedAnswerSha256: string;
|
|
3114
|
+
baseline: {
|
|
3115
|
+
answer: LoComoRecallTextDigest;
|
|
3116
|
+
recall: LoComoRecallContextSummary;
|
|
3117
|
+
};
|
|
3118
|
+
real: {
|
|
3119
|
+
answer: LoComoRecallTextDigest;
|
|
3120
|
+
recall: LoComoRecallContextSummary;
|
|
3121
|
+
};
|
|
3122
|
+
displacedLines: LoComoRecallLineDelta;
|
|
3123
|
+
introducedLines: LoComoRecallLineDelta;
|
|
3124
|
+
}
|
|
3125
|
+
interface LoComoRecallResultProvenance {
|
|
3126
|
+
reference: string;
|
|
3127
|
+
sha256: string;
|
|
3128
|
+
resultId: string;
|
|
3129
|
+
gitSha: string;
|
|
3130
|
+
remnicVersion: string;
|
|
3131
|
+
runtimeProfile: "baseline" | "real";
|
|
3132
|
+
systemProvider: string;
|
|
3133
|
+
systemModel: string;
|
|
3134
|
+
judgeProvider: string;
|
|
3135
|
+
judgeModel: string;
|
|
3136
|
+
seeds: number[];
|
|
3137
|
+
taskPayloadSha256: string;
|
|
3138
|
+
}
|
|
3139
|
+
interface LoComoRecallDeltaReport {
|
|
3140
|
+
schemaVersion: 1;
|
|
3141
|
+
benchmarkId: "locomo";
|
|
3142
|
+
comparison: {
|
|
3143
|
+
baseline: LoComoRecallResultProvenance;
|
|
3144
|
+
real: LoComoRecallResultProvenance;
|
|
3145
|
+
};
|
|
3146
|
+
taskCount: number;
|
|
3147
|
+
primaryMetric: string;
|
|
3148
|
+
overall: LoComoRecallMetricDelta;
|
|
3149
|
+
categories: LoComoRecallCategoryDelta[];
|
|
3150
|
+
topRegressions: LoComoFinalContextRegression[];
|
|
3151
|
+
evidenceBoundary: {
|
|
3152
|
+
finalContextComparison: "complete";
|
|
3153
|
+
retrievalTierAttribution: "unavailable-in-cached-results";
|
|
3154
|
+
hiddenEvidenceUsed: false;
|
|
3155
|
+
explanation: string;
|
|
3156
|
+
};
|
|
3157
|
+
}
|
|
3158
|
+
interface DiagnoseLoComoRecallDeltaOptions {
|
|
3159
|
+
baseline: LoComoRawResultEvidence;
|
|
3160
|
+
real: LoComoRawResultEvidence;
|
|
3161
|
+
primaryMetric?: string;
|
|
3162
|
+
maxRegressions?: number;
|
|
3163
|
+
}
|
|
3164
|
+
/**
|
|
3165
|
+
* Return a stable provenance label without exposing the caller's directory
|
|
3166
|
+
* layout. CLI callers should use this instead of persisting an input path.
|
|
3167
|
+
*/
|
|
3168
|
+
declare function sanitizeLoComoResultReference(path: string): string;
|
|
3169
|
+
declare function diagnoseLoComoRecallDelta(options: DiagnoseLoComoRecallDeltaOptions): LoComoRecallDeltaReport;
|
|
3170
|
+
declare function renderLoComoRecallDeltaMarkdown(report: LoComoRecallDeltaReport): string;
|
|
3171
|
+
|
|
3026
3172
|
/**
|
|
3027
3173
|
* Dataset-contamination guard.
|
|
3028
3174
|
*
|
|
@@ -3879,7 +4025,7 @@ interface RunJudgeCalibrationOptions {
|
|
|
3879
4025
|
* binning function so they are compared on the same scale.
|
|
3880
4026
|
*/
|
|
3881
4027
|
binScore?: (score: number) => JudgeCategory;
|
|
3882
|
-
/** Override the slice size (default
|
|
4028
|
+
/** Override the slice size (default 200; mainly for tests). */
|
|
3883
4029
|
sliceSize?: number;
|
|
3884
4030
|
/** Override the warning threshold (default 0.7). */
|
|
3885
4031
|
threshold?: number;
|
|
@@ -5078,4 +5224,4 @@ declare function checkCodingGraphRegression(report: CodingGraphBenchReport, base
|
|
|
5078
5224
|
*/
|
|
5079
5225
|
declare function buildBaselineFromReport(report: CodingGraphBenchReport, note: string): CodingGraphBaseline;
|
|
5080
5226
|
|
|
5081
|
-
export { AMA_BENCH_DIAGNOSTIC_VARIANTS, ASSISTANT_AGENT_CONFIG_KEY, ASSISTANT_JUDGE_CONFIG_KEY, ASSISTANT_MEETING_PREP_SCENARIOS, ASSISTANT_MEETING_PREP_SMOKE_SCENARIOS, ASSISTANT_MORNING_BRIEF_SCENARIOS, ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SMOKE_SCENARIOS, ASSISTANT_RUBRIC_DIMENSIONS, ASSISTANT_RUBRIC_ID_KEY, ASSISTANT_SEEDS_CONFIG_KEY, ASSISTANT_SPOT_CHECK_DIR_KEY, ASSISTANT_SYNTHESIS_SCENARIOS, ASSISTANT_SYNTHESIS_SMOKE_SCENARIOS, type AblationConfigOverrides, type AbstentionRetrievalCase, type AggregateMetrics, type AmaBenchDiagnosticAdapterOptions, type AmaBenchDiagnosticAnswererMode, type AmaBenchDiagnosticBreakdown, type AmaBenchDiagnosticMatrixArtifact, type AmaBenchDiagnosticRecallMode, type AmaBenchDiagnosticRunContext, type AmaBenchDiagnosticTaskEvidence, type AmaBenchDiagnosticTaskRow, type AmaBenchDiagnosticVariant, type AmaBenchDiagnosticVariantSummary, type AnthropicProviderConfig, type AssistantAgent, type AssistantMemoryFact, type AssistantMemoryGraph, type AssistantRubricDimension, type AssistantRubricRequest, type AssistantRubricScores, type AssistantRunnerOptions, type AssistantScenario, type AssistantStance, type AttackRecallOptions, type AttackRetrievalHit, type AttackerMode, BENCHMARK_ARTIFACT_SCHEMA_VERSION, BENCHMARK_INTEGRITY_META_SCHEMA, BENCHMARK_REPRO_MANIFEST_FILENAME, BENCHMARK_REPRO_MANIFEST_SCHEMA_VERSION, BENCHMARK_RESULT_SCHEMA, BENCHMARK_SPLIT_TYPES, type BaselineRow, type BaselineScenario, type BeamDatasetPreview, type BenchConfig, type BenchJudge, type BenchJudgeResult, type BenchMemoryAdapter, type BenchModelSource, type BenchReasoningEffort, type BenchRecallOptions, type BenchRecallSupportAssessment, type BenchRecallSupportRequest, type BenchRecallSupportStatus, type BenchResponder, type BenchResponse, type BenchRuntimeProfile, type BenchTier, type BenchmarkArtifact, type BenchmarkArtifactEnvironment, type BenchmarkArtifactHardware, type BenchmarkArtifactJudgeCalibration, type BenchmarkArtifactPerTaskScore, type BenchmarkArtifactSystem, type BenchmarkArtifactTier, type BenchmarkCategory, type BenchmarkDefinition, type BenchmarkIntegrityMeta, type BenchmarkMeta, type BenchmarkMode, type BenchmarkReport, type BenchmarkReproManifest, type BenchmarkReproManifestDataset, type BenchmarkReproManifestFile, type BenchmarkReproManifestResult, type BenchmarkResult, type BenchmarkSplitType, type BenchmarkStatus, type BenchmarkSuiteResult, type BenchmarkTier, type BootstrapKappaOptions, type BootstrapKappaResult, type BuildBenchmarkArtifactInput, type BuildBenchmarkPublishFeedOptions, type BuildBenchmarkReproManifestOptions, type BuiltInProvider, CALIBRATION_SLICE_SIZE, CANARY_FIXED_RECALL, CANARY_SCORE_FLOOR, DEFAULT_10K_FIXTURE as CODING_GRAPH_10K_FIXTURE, CODING_GRAPH_BENCH_SCHEMA_VERSION, DEFAULT_TOLERANCE_PERCENT as CODING_GRAPH_DEFAULT_TOLERANCE, MIN_ITERATIONS as CODING_GRAPH_MIN_ITERATIONS, DEFAULT_SMOKE_FIXTURE as CODING_GRAPH_SMOKE_FIXTURE, type CalibrationAnswer, type CalibrationVerdictPair, type CanaryAdapterOptions, type CanaryFloorCheck, type ClaudeCliProviderConfig, type CodexCliProviderConfig, type CodingGraphBaseline, type CodingGraphBenchConfig, type CodingGraphBenchReport, type MachineFingerprint as CodingGraphMachineFingerprint, type CodingGraphMetricKey, type RegressionMetricDetail as CodingGraphRegressionDetail, type RegressionMetricKey as CodingGraphRegressionKey, type RegressionGateResult as CodingGraphRegressionResult, type CohenKappaResult, type ComparisonMetricDelta, type ComparisonResult, type CompletionOpts, type CompletionResult, type ConfidenceInterval, type ContaminationCheckResult, type ContaminationEntry, type ContaminationManifest, type CustomBenchmarkScoring, type CustomBenchmarkSpec, type CustomBenchmarkTask, DEFAULT_ABLATION_BENCHMARK, DEFAULT_ABLATION_BOOTSTRAP_SEED, DEFAULT_ASSISTANT_RUBRIC_ID, DEFAULT_BASELINE_SCENARIOS, DEFAULT_JUDGE_BINARIZATION_THRESHOLD, DEFAULT_KAPPA_BOOTSTRAP_SAMPLES, DEFAULT_KAPPA_CONFIDENCE_LEVEL, DEFAULT_OPENAI_RESPONSES_JUDGE_MODEL, type DatasetSource, type DiagnoseLoComoProfileDeltaOptions, type DiscoveredModel, EMPTY_CONTAMINATION_MANIFEST, type EffectSizeInterpretation, type EffectSizeSummary, type ExplainResult, type ExtractedEntity, type ExtractedLink, type ExtractedPage, type ExtractionAttackOptions, type ExtractionAttackResult, type ExtractionAttackTarget, type FixtureGenerator, type FixtureOutput, type FixtureVariant, GENERAL_ANSWER_JUDGE_RUBRIC, type GeneratedFile, type GeneratedRepo, type GoldEntity, type GoldEntityType, type GoldGraph, type GoldLink, type GoldPage, type HarnessRng, INTEGRITY_CIPHER_ALGORITHM, INTEGRITY_HASH_ALGORITHM, INTEGRITY_META_FIELDS, type IngestionBenchAdapter, type IngestionLog, JUDGE_CALIBRATION_KAPPA_THRESHOLD, type JudgeCalibrationIdentities, type JudgeCalibrationResult, type JudgeCategory, type KappaConfidenceInterval, LOCAL_LAB_PROVIDER_KINDS, LOCOMO_DATASET_FILENAMES, LONG_MEM_EVAL_DATASET_FILENAMES, type LeaderboardArtifactWrite, type LettaAdapterConfig, LettaMemCorrectAdapter, type LlmJudge, type LlmProvider, type LoComoCategoryDelta, type LoComoMetricDelta, type LoComoProfileArtifactEvidence, type LoComoProfileDeltaReport, type LoComoTaskRegression, type LoadDatasetOptions, type LoadSealedQrelsOptions, type LoadedDataset, type LoadedJudgeCalibrationState, type LocalLabManifest, type LocalLabManifestNotes, type LocalLabPhase, type LocalLabPhaseDescriptor, type LocalLabPhaseExecute, type LocalLabPhaseName, type LocalLabPhaseOutcome, LocalLabPreflightError, type LocalLabPreflightFailure, type LocalLabPreflightInput, type LocalLabPreflightOptions, type LocalLabPreflightResult, type LocalLabPreflightSuccess, type LocalLabProviderKind, type LocalLabRoleConfig, type LocalLlmProviderConfig, MEMCORRECT_CORRECTION_ACCEPTANCE_RUBRIC, MEMCORRECT_CORRECTION_ACCEPTANCE_RUBRIC_VERSION, MEMCORRECT_STALE_HARM_RUBRIC, MEMCORRECT_STALE_HARM_RUBRIC_VERSION, MEMORY_EVAL_DIMENSIONS, MEMORY_EVAL_PUBLIC_LINE, MIN_CALIBRATION_SOURCE_TASKS, MITIGATED_BASELINE_SCENARIOS, type McpArgumentSemantic, type McpBackendErrorCode, type McpBackendResult, type McpBenchMemoryAdapter, type McpConformanceResult, type McpHttpTransportConfig, type McpListedTool, type McpMemCorrectAdapter, type McpMemoryAdapterOptions, McpMemoryBackendError, type McpMemoryToolMapping, type McpMemoryTransportConfig, type McpStdioTransportConfig, type McpToolCallResult, type McpToolClient, type McpToolMappingEntry, type McpToolMappingValue, type McpToolOperation, type Mem0AdapterConfig, Mem0MemCorrectAdapter, type MemCorrectGeneratorOptions, type MemCorrectJudgeRequest, type MemCorrectJudgeResult, type MemCorrectSystemAdapter, type MemoryEvalCategory, type MemoryEvalDimension, type MemoryEvalDimensionId, type MemoryEvalMetric, type MemoryGraph, type MemoryStats, type MemorySystem, type Message, type MetricAggregate, type MicroMetric, MissingCredentialError, type MitigatedBaselineConfig, type MitigatedTargetConfig, type MultipleChoiceQuestion, OPENAI_RESPONSES_JUDGE_RUBRIC_VERSION, OTHER_NAMESPACE_MEMORIES, type OllamaProviderConfig, type OpenAiCompatibleProviderConfig, OpenAiResponsesJudgeError, type OpenAiResponsesJudgeErrorCode, type OpenAiResponsesJudgeTelemetry, OpenAiResponsesProvider, type OpenAiResponsesProviderConfig, type OpenAiResponsesVerdict, type OpenAiResponsesVerdictResult, PROCEDURAL_REAL_SCENARIOS, PROCEDURAL_REAL_SCENARIOS_SMOKE, PUBLISHED_BENCHMARK_ARTIFACT_IDS, type PersonalizationRetrievalCase, type PreflightDiscoveredModel, type ProceduralAblationArtifact, type ProceduralAblationPerCase, type ProceduralAblationScenario, type ProceduralRealScenario, type ProceduralRealScenarioCategory, type ProviderBaseConfig, type ProviderConfig, type ProviderDiscoveryResult, type ProviderFactoryConfig, type PublishSkipReason, type PublishSkipRecord, type PublishedBenchmarkFeed, type PublishedBenchmarkFeedEntry, type PublishedBenchmarkId, REQUIRED_FRONTMATTER_FIELDS, type RecallMetrics, type RecoveredMemory, type RegressionDetail, type RegressionGateResult$1 as RegressionGateResult, type RemnicAdapterOptions, type ReportCardProvenanceContext, type ResolveBenchRuntimeProfileOptions, type ResolvedBenchRuntimeProfile, type ResolvedLocalLabProfile, type ResolvedLocalLabRole, type ResolvedRunBenchmarkOptions, type RotatedChoices, type RunBenchmarkOptions, type RunJudgeCalibrationOptions, type RunProceduralAblationCliArgs, type RunProceduralAblationOptions, type RunSequentialPhasesOptions, SCHEMA_TIER_FIXTURE, SCHEMA_TIER_SMOKE_FIXTURE, SEALED_PROMPT_REGISTRY, SINGLE_FLAG_ABLATION_MATRIX, SYNTHETIC_MEMORIES, type SanitizedDiagnosticProvider, type SavedBaseline, type SchemaTierCorpus, type SchemaTierFixture, type SchemaTierName, type SchemaTierPage, type SchemaTierPageFrontmatter, type SealedArtifact, type SealedJudgeDecision, type SealedJudgeInput, type SealedQrelsArtifact, type SealedQrelsHandle, type SealedRubric, type SearchResult, type SeededMemory, type SeededRng, type SequentialPhaseHooks, type SingleFlagAblationCell, type SingleFlagAblationId, type SpotCheckLogger, type StatisticalReport, type StructuredJudge, StructuredJudgeError, type StructuredJudgeErrorCode, type StructuredJudgeProvider, type StructuredJudgeTelemetry, type StructuredJudgeVerdict, type StructuredJudgeVerdictResult, type StructuredVerdictRequest, type SyntheticEdge, type SyntheticEmailIngestionAdapterOptions, type SyntheticFileIR, type SyntheticRepoConfig, type SyntheticSymbol, type SyntheticTargetOptions, type TaskResult, type TaskTokenUsage, type TemporalRetrievalCase, type ThirdPartyAdapterConfig, type TierDetail, type TimelineEntry, type TokenUsage, type WallMetric, type WriteBenchmarkArtifactResult, type ZepAdapterConfig, ZepMemCorrectAdapter, addContaminationEntry, aggregateTaskScores, answerBenchmarkQuestion, assertCanaryUnderFloor, assertIntegrityMetaPresent, assertPublishableIntegrity, assertSha256Hex, assistantMeetingPrepDefinition, assistantMorningBriefDefinition, assistantNextBestActionDefinition, assistantSynthesisDefinition, backlinkF1, binarizeJudgeScore, bootstrapCohensKappaConfidenceInterval, bootstrapMeanConfidenceInterval, buildAmaBenchDiagnosticMatrixArtifact, buildAmaBenchDiagnosticVariantSummary, buildAmaBenchLeaderboardRows, buildBaselineFromReport, buildBenchmarkArtifact, buildBenchmarkArtifactFilename, buildBenchmarkPublishFeed, buildBenchmarkReproManifest, buildBenchmarkRunSeeds, buildJudgePayload, buildOracleTrajectoryRecall, buildSchemaTierFixture, buildSchemaTierSmokeFixture, calendarFixture, canonicalJsonStringify, captureMachineFingerprint, chatFixture, checkCodingGraphRegression, checkDatasetContamination, checkRegression, clampScore, cohensD, compareResults, computeCohensKappa, computeSealHash, containsAnswer, createSeededRng$1 as createAdamSeededRng, createAmaBenchDiagnosticAdapter, createAnthropicProvider, createCanaryAdapter, createClaudeCliProvider, createCodexCliProvider, createSeededRng as createCodingGraphSeededRng, createDeterministicSpotCheckLogger, createGatewayResponder, createLightweightAdapter, createLiteLlmProvider, createLocalLlmProvider, createMcpDemoMemCorrectAdapter, createMcpDemoMemoryAdapter, createMcpMemCorrectAdapter, createMcpMemoryAdapter, createMitigatedTarget, createOllamaProvider, createOpenAiCompatibleProvider, createOpenAiResponsesBenchJudge, createOpenAiResponsesProvider, createSeededRandom as createProceduralAblationSeededRandom, createProvider, createProviderBackedAmaBenchRecommendedJudge, createProviderBackedJudge, createProviderBackedResponder, createProviderBackedStructuredJudge, createRemnicAdapter, createResponderFromProvider, createSeededRng$2 as createSeededRng, createSpotCheckFileLogger, createStructuredBenchJudge, createStructuredJudgeFromProvider, createSyntheticEmailIngestionAdapter, createSyntheticTarget, createTimeoutGuardedAdapter, defaultBenchmarkBaselineDir, defaultBenchmarkPublishPath, deleteBenchmarkResults, diagnoseLoComoProfileDelta, discoverAllProviders, discoveryEndpointFor, emailFixture, entityRecall, exactMatch, extractMetrics as extractCodingGraphMetrics, extractMarkdownSectionsByTitle, f1Score, fixtureToAblationScenarios, formatHandoffNote, formatMissingDatasetError, generateReport, generateSyntheticRepo, getAblationCell, getBenchmark, getBenchmarkLowerIsBetter, getMemoryEvalDimension, getRemnicVersion, hashBenchmarkArtifact, hashBytes, hashCanonicalJson, hashString, integrityMetaIsComplete, interpretEffectSize, isAmaBenchUnknownLikeAnswer, isContaminationEntry, isContaminationManifest, isSealedQrelsArtifact, isSha256Hex, isStructuredJudgeProvider, judgeMemCorrectCorrectionAcceptance, judgeMemCorrectStaleMemoryHarm, linkMatches, listBenchmarkBaselines, listBenchmarkResults, listBenchmarks, listMemoryEvalBenchmarkIds, listMemoryEvalDimensions, llmJudgeScore, llmJudgeScoreDetailed, loadAblationFixture, loadBaseline, loadBeamDatasetPreview, loadBenchmarkArtifact, loadBenchmarkBaseline, loadBenchmarkReportCardProvenance, loadBenchmarkResult, loadCustomBenchmarkFile, loadJudgeCalibrationState, loadLoCoMo10, loadLocalLabManifest, loadLongMemEvalS, loadSealKeyFromEnv, loadSealedQrels, loadSealedRubric, matchEntity, mergeContaminationManifests, openSeal, orchestrateBenchmarkRuns, pairedDeltaConfidenceInterval, parseBenchmarkArtifact, parseCustomBenchmark, parseLocalLabManifest, parseRubricResponse, parseSealedQrels, pickStableQualifiedName, precisionAtK, preflightLocalLabRole, projectFolderFixture, recallAtK, redactBenchmarkResultSecrets, renderBaselineMarkdown, renderBenchmarkResultExport, renderLoComoProfileDeltaMarkdown, renderMemorySummaryForJudge, renderMemoryViewForAgent, resolveAssistantAgent, resolveAssistantRubricId, resolveAssistantSeeds, resolveAssistantSpotCheckDir, resolveBenchRuntimeProfile, resolveBenchmarkPhaseTimeoutMs, resolveBenchmarkProgressLogging, resolveBenchmarkResultReference, resolveBenchmarkRunCount, resolveLocalLabProfile, resolveLocalLabRole, resolveStructuredJudge, rotateDistractors, rougeL, runAssistantBenchmark, runAssistantMeetingPrepBenchmark, runAssistantMorningBriefBenchmark, runAssistantNextBestActionBenchmark, runAssistantSynthesisBenchmark, runBaseline, runBenchSuite, runBenchmark, runCodingGraphBenchmark, runCustomBenchmarkFile, runExplain, runExtractionAttack, runJudgeCalibration, runMitigatedBaseline, runProceduralAblation, runProceduralAblationCli, runSealedJudge, runSequentialPhases, safeHexEqual, saveBaseline, saveBenchmarkBaseline, schemaCompleteness, sealPayload, selectAmaBenchDiagnosticVariants, selectCalibrationSlice, selectFixtureVariant, serializeBenchmarkArtifact, serializeJsonl, serializeSealedQrels, shuffleTasks, timed, verifyRubricDigest, writeBenchmarkArtifact, writeBenchmarkPublishFeed, writeBenchmarkReproManifest, writeBenchmarkResult, writeJudgeCalibrationState, writeLeaderboardArtifactsForResult, zeroScores };
|
|
5227
|
+
export { AMA_BENCH_DIAGNOSTIC_VARIANTS, ASSISTANT_AGENT_CONFIG_KEY, ASSISTANT_JUDGE_CONFIG_KEY, ASSISTANT_MEETING_PREP_SCENARIOS, ASSISTANT_MEETING_PREP_SMOKE_SCENARIOS, ASSISTANT_MORNING_BRIEF_SCENARIOS, ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SMOKE_SCENARIOS, ASSISTANT_RUBRIC_DIMENSIONS, ASSISTANT_RUBRIC_ID_KEY, ASSISTANT_SEEDS_CONFIG_KEY, ASSISTANT_SPOT_CHECK_DIR_KEY, ASSISTANT_SYNTHESIS_SCENARIOS, ASSISTANT_SYNTHESIS_SMOKE_SCENARIOS, type AblationConfigOverrides, type AbstentionRetrievalCase, type AggregateMetrics, type AmaBenchDiagnosticAdapterOptions, type AmaBenchDiagnosticAnswererMode, type AmaBenchDiagnosticBreakdown, type AmaBenchDiagnosticMatrixArtifact, type AmaBenchDiagnosticRecallMode, type AmaBenchDiagnosticRunContext, type AmaBenchDiagnosticTaskEvidence, type AmaBenchDiagnosticTaskRow, type AmaBenchDiagnosticVariant, type AmaBenchDiagnosticVariantSummary, type AnthropicProviderConfig, type AssistantAgent, type AssistantMemoryFact, type AssistantMemoryGraph, type AssistantRubricDimension, type AssistantRubricRequest, type AssistantRubricScores, type AssistantRunnerOptions, type AssistantScenario, type AssistantStance, type AttackRecallOptions, type AttackRetrievalHit, type AttackerMode, BENCHMARK_ARTIFACT_SCHEMA_VERSION, BENCHMARK_INTEGRITY_META_SCHEMA, BENCHMARK_REPRO_MANIFEST_FILENAME, BENCHMARK_REPRO_MANIFEST_SCHEMA_VERSION, BENCHMARK_RESULT_SCHEMA, BENCHMARK_SPLIT_TYPES, type BaselineRow, type BaselineScenario, type BeamDatasetPreview, type BenchConfig, type BenchJudge, type BenchJudgeResult, type BenchMemoryAdapter, type BenchModelSource, type BenchReasoningEffort, type BenchRecallOptions, type BenchRecallSupportAssessment, type BenchRecallSupportRequest, type BenchRecallSupportStatus, type BenchResponder, type BenchResponse, type BenchRuntimeProfile, type BenchTier, type BenchmarkArtifact, type BenchmarkArtifactEnvironment, type BenchmarkArtifactHardware, type BenchmarkArtifactJudgeCalibration, type BenchmarkArtifactPerTaskScore, type BenchmarkArtifactSystem, type BenchmarkArtifactTier, type BenchmarkCategory, type BenchmarkDefinition, type BenchmarkIntegrityMeta, type BenchmarkMeta, type BenchmarkMode, type BenchmarkReport, type BenchmarkReproManifest, type BenchmarkReproManifestDataset, type BenchmarkReproManifestFile, type BenchmarkReproManifestResult, type BenchmarkResult, type BenchmarkSplitType, type BenchmarkStatus, type BenchmarkSuiteResult, type BenchmarkTier, type BootstrapKappaOptions, type BootstrapKappaResult, type BuildBenchmarkArtifactInput, type BuildBenchmarkPublishFeedOptions, type BuildBenchmarkReproManifestOptions, type BuiltInProvider, CALIBRATION_SLICE_SIZE, CANARY_FIXED_RECALL, CANARY_SCORE_FLOOR, DEFAULT_10K_FIXTURE as CODING_GRAPH_10K_FIXTURE, CODING_GRAPH_BENCH_SCHEMA_VERSION, DEFAULT_TOLERANCE_PERCENT as CODING_GRAPH_DEFAULT_TOLERANCE, MIN_ITERATIONS as CODING_GRAPH_MIN_ITERATIONS, DEFAULT_SMOKE_FIXTURE as CODING_GRAPH_SMOKE_FIXTURE, type CalibrationAnswer, type CalibrationVerdictPair, type CanaryAdapterOptions, type CanaryFloorCheck, type ClaudeCliProviderConfig, type CodexCliProviderConfig, type CodingGraphBaseline, type CodingGraphBenchConfig, type CodingGraphBenchReport, type MachineFingerprint as CodingGraphMachineFingerprint, type CodingGraphMetricKey, type RegressionMetricDetail as CodingGraphRegressionDetail, type RegressionMetricKey as CodingGraphRegressionKey, type RegressionGateResult as CodingGraphRegressionResult, type CohenKappaResult, type ComparisonMetricDelta, type ComparisonResult, type CompletionOpts, type CompletionResult, type ConfidenceInterval, type ContaminationCheckResult, type ContaminationEntry, type ContaminationManifest, type CustomBenchmarkScoring, type CustomBenchmarkSpec, type CustomBenchmarkTask, DEFAULT_ABLATION_BENCHMARK, DEFAULT_ABLATION_BOOTSTRAP_SEED, DEFAULT_ASSISTANT_RUBRIC_ID, DEFAULT_BASELINE_SCENARIOS, DEFAULT_JUDGE_BINARIZATION_THRESHOLD, DEFAULT_KAPPA_BOOTSTRAP_SAMPLES, DEFAULT_KAPPA_CONFIDENCE_LEVEL, DEFAULT_OPENAI_RESPONSES_JUDGE_MODEL, type DatasetSource, type DiagnoseLoComoProfileDeltaOptions, type DiagnoseLoComoRecallDeltaOptions, type DiscoveredModel, EMPTY_CONTAMINATION_MANIFEST, type EffectSizeInterpretation, type EffectSizeSummary, type ExplainResult, type ExtractedEntity, type ExtractedLink, type ExtractedPage, type ExtractionAttackOptions, type ExtractionAttackResult, type ExtractionAttackTarget, type FixtureGenerator, type FixtureOutput, type FixtureVariant, GENERAL_ANSWER_JUDGE_RUBRIC, type GeneratedFile, type GeneratedRepo, type GoldEntity, type GoldEntityType, type GoldGraph, type GoldLink, type GoldPage, type HarnessRng, INTEGRITY_CIPHER_ALGORITHM, INTEGRITY_HASH_ALGORITHM, INTEGRITY_META_FIELDS, type IngestionBenchAdapter, type IngestionLog, JUDGE_CALIBRATION_KAPPA_THRESHOLD, type JudgeCalibrationIdentities, type JudgeCalibrationResult, type JudgeCategory, type KappaConfidenceInterval, LOCAL_LAB_PROVIDER_KINDS, LOCOMO_DATASET_FILENAMES, LOCOMO_FULL_TASK_COUNT, LOCOMO_RECALL_DIFF_LINE_LIMIT, LOCOMO_RECALL_EXCERPT_CHARS, LONG_MEM_EVAL_DATASET_FILENAMES, type LeaderboardArtifactWrite, type LettaAdapterConfig, LettaMemCorrectAdapter, type LlmJudge, type LlmProvider, type LoComoCategoryDelta, type LoComoFinalContextRegression, type LoComoMetricDelta, type LoComoProfileArtifactEvidence, type LoComoProfileDeltaReport, type LoComoRawResultEvidence, type LoComoRecallCategoryDelta, type LoComoRecallContextSummary, type LoComoRecallDeltaReport, type LoComoRecallLineDelta, type LoComoRecallLineEvidence, type LoComoRecallMetricDelta, type LoComoRecallResultProvenance, type LoComoRecallTextDigest, type LoComoTaskRegression, type LoadDatasetOptions, type LoadSealedQrelsOptions, type LoadedDataset, type LoadedJudgeCalibrationState, type LocalLabManifest, type LocalLabManifestNotes, type LocalLabPhase, type LocalLabPhaseDescriptor, type LocalLabPhaseExecute, type LocalLabPhaseName, type LocalLabPhaseOutcome, LocalLabPreflightError, type LocalLabPreflightFailure, type LocalLabPreflightInput, type LocalLabPreflightOptions, type LocalLabPreflightResult, type LocalLabPreflightSuccess, type LocalLabProviderKind, type LocalLabRoleConfig, type LocalLlmProviderConfig, MEMCORRECT_CORRECTION_ACCEPTANCE_RUBRIC, MEMCORRECT_CORRECTION_ACCEPTANCE_RUBRIC_VERSION, MEMCORRECT_STALE_HARM_RUBRIC, MEMCORRECT_STALE_HARM_RUBRIC_VERSION, MEMORY_EVAL_DIMENSIONS, MEMORY_EVAL_PUBLIC_LINE, MIN_CALIBRATION_SOURCE_TASKS, MITIGATED_BASELINE_SCENARIOS, type McpArgumentSemantic, type McpBackendErrorCode, type McpBackendResult, type McpBenchMemoryAdapter, type McpConformanceResult, type McpHttpTransportConfig, type McpListedTool, type McpMemCorrectAdapter, type McpMemoryAdapterOptions, McpMemoryBackendError, type McpMemoryToolMapping, type McpMemoryTransportConfig, type McpStdioTransportConfig, type McpToolCallResult, type McpToolClient, type McpToolMappingEntry, type McpToolMappingValue, type McpToolOperation, type Mem0AdapterConfig, Mem0MemCorrectAdapter, type MemCorrectGeneratorOptions, type MemCorrectJudgeRequest, type MemCorrectJudgeResult, type MemCorrectSystemAdapter, type MemoryEvalCategory, type MemoryEvalDimension, type MemoryEvalDimensionId, type MemoryEvalMetric, type MemoryGraph, type MemoryStats, type MemorySystem, type Message, type MetricAggregate, type MicroMetric, MissingCredentialError, type MitigatedBaselineConfig, type MitigatedTargetConfig, type MultipleChoiceQuestion, OPENAI_RESPONSES_JUDGE_RUBRIC_VERSION, OTHER_NAMESPACE_MEMORIES, type OllamaProviderConfig, type OpenAiCompatibleProviderConfig, OpenAiResponsesJudgeError, type OpenAiResponsesJudgeErrorCode, type OpenAiResponsesJudgeTelemetry, OpenAiResponsesProvider, type OpenAiResponsesProviderConfig, type OpenAiResponsesVerdict, type OpenAiResponsesVerdictResult, PROCEDURAL_REAL_SCENARIOS, PROCEDURAL_REAL_SCENARIOS_SMOKE, PUBLISHED_BENCHMARK_ARTIFACT_IDS, type PersonalizationRetrievalCase, type PreflightDiscoveredModel, type ProceduralAblationArtifact, type ProceduralAblationPerCase, type ProceduralAblationScenario, type ProceduralRealScenario, type ProceduralRealScenarioCategory, type ProviderBaseConfig, type ProviderConfig, type ProviderDiscoveryResult, type ProviderFactoryConfig, type PublishSkipReason, type PublishSkipRecord, type PublishedBenchmarkFeed, type PublishedBenchmarkFeedEntry, type PublishedBenchmarkId, REQUIRED_FRONTMATTER_FIELDS, type RecallMetrics, type RecoveredMemory, type RegressionDetail, type RegressionGateResult$1 as RegressionGateResult, type RemnicAdapterOptions, type ReportCardProvenanceContext, type ResolveBenchRuntimeProfileOptions, type ResolvedBenchRuntimeProfile, type ResolvedLocalLabProfile, type ResolvedLocalLabRole, type ResolvedRunBenchmarkOptions, type RotatedChoices, type RunBenchmarkOptions, type RunJudgeCalibrationOptions, type RunProceduralAblationCliArgs, type RunProceduralAblationOptions, type RunSequentialPhasesOptions, SCHEMA_TIER_FIXTURE, SCHEMA_TIER_SMOKE_FIXTURE, SEALED_PROMPT_REGISTRY, SINGLE_FLAG_ABLATION_MATRIX, SYNTHETIC_MEMORIES, type SanitizedDiagnosticProvider, type SavedBaseline, type SchemaTierCorpus, type SchemaTierFixture, type SchemaTierName, type SchemaTierPage, type SchemaTierPageFrontmatter, type SealedArtifact, type SealedJudgeDecision, type SealedJudgeInput, type SealedQrelsArtifact, type SealedQrelsHandle, type SealedRubric, type SearchResult, type SeededMemory, type SeededRng, type SequentialPhaseHooks, type SingleFlagAblationCell, type SingleFlagAblationId, type SpotCheckLogger, type StatisticalReport, type StructuredJudge, StructuredJudgeError, type StructuredJudgeErrorCode, type StructuredJudgeProvider, type StructuredJudgeTelemetry, type StructuredJudgeVerdict, type StructuredJudgeVerdictResult, type StructuredVerdictRequest, type SyntheticEdge, type SyntheticEmailIngestionAdapterOptions, type SyntheticFileIR, type SyntheticRepoConfig, type SyntheticSymbol, type SyntheticTargetOptions, type TaskResult, type TaskTokenUsage, type TemporalRetrievalCase, type ThirdPartyAdapterConfig, type TierDetail, type TimelineEntry, type TokenUsage, type WallMetric, type WriteBenchmarkArtifactResult, type ZepAdapterConfig, ZepMemCorrectAdapter, addContaminationEntry, aggregateTaskScores, answerBenchmarkQuestion, assertCanaryUnderFloor, assertIntegrityMetaPresent, assertPublishableIntegrity, assertSha256Hex, assistantMeetingPrepDefinition, assistantMorningBriefDefinition, assistantNextBestActionDefinition, assistantSynthesisDefinition, backlinkF1, binarizeJudgeScore, bootstrapCohensKappaConfidenceInterval, bootstrapMeanConfidenceInterval, buildAmaBenchDiagnosticMatrixArtifact, buildAmaBenchDiagnosticVariantSummary, buildAmaBenchLeaderboardRows, buildBaselineFromReport, buildBenchmarkArtifact, buildBenchmarkArtifactFilename, buildBenchmarkPublishFeed, buildBenchmarkReproManifest, buildBenchmarkRunSeeds, buildJudgePayload, buildOracleTrajectoryRecall, buildSchemaTierFixture, buildSchemaTierSmokeFixture, calendarFixture, canonicalJsonStringify, captureMachineFingerprint, chatFixture, checkCodingGraphRegression, checkDatasetContamination, checkRegression, clampScore, cohensD, compareResults, computeCohensKappa, computeSealHash, containsAnswer, createSeededRng$1 as createAdamSeededRng, createAmaBenchDiagnosticAdapter, createAnthropicProvider, createCanaryAdapter, createClaudeCliProvider, createCodexCliProvider, createSeededRng as createCodingGraphSeededRng, createDeterministicSpotCheckLogger, createGatewayResponder, createLightweightAdapter, createLiteLlmProvider, createLocalLlmProvider, createMcpDemoMemCorrectAdapter, createMcpDemoMemoryAdapter, createMcpMemCorrectAdapter, createMcpMemoryAdapter, createMitigatedTarget, createOllamaProvider, createOpenAiCompatibleProvider, createOpenAiResponsesBenchJudge, createOpenAiResponsesProvider, createSeededRandom as createProceduralAblationSeededRandom, createProvider, createProviderBackedAmaBenchRecommendedJudge, createProviderBackedJudge, createProviderBackedResponder, createProviderBackedStructuredJudge, createRemnicAdapter, createResponderFromProvider, createSeededRng$2 as createSeededRng, createSpotCheckFileLogger, createStructuredBenchJudge, createStructuredJudgeFromProvider, createSyntheticEmailIngestionAdapter, createSyntheticTarget, createTimeoutGuardedAdapter, defaultBenchmarkBaselineDir, defaultBenchmarkPublishPath, deleteBenchmarkResults, diagnoseLoComoProfileDelta, diagnoseLoComoRecallDelta, discoverAllProviders, discoveryEndpointFor, emailFixture, entityRecall, exactMatch, extractMetrics as extractCodingGraphMetrics, extractMarkdownSectionsByTitle, f1Score, fixtureToAblationScenarios, formatHandoffNote, formatMissingDatasetError, generateReport, generateSyntheticRepo, getAblationCell, getBenchmark, getBenchmarkLowerIsBetter, getMemoryEvalDimension, getRemnicVersion, hashBenchmarkArtifact, hashBytes, hashCanonicalJson, hashString, integrityMetaIsComplete, interpretEffectSize, isAmaBenchUnknownLikeAnswer, isContaminationEntry, isContaminationManifest, isSealedQrelsArtifact, isSha256Hex, isStructuredJudgeProvider, judgeMemCorrectCorrectionAcceptance, judgeMemCorrectStaleMemoryHarm, linkMatches, listBenchmarkBaselines, listBenchmarkResults, listBenchmarks, listMemoryEvalBenchmarkIds, listMemoryEvalDimensions, llmJudgeScore, llmJudgeScoreDetailed, loadAblationFixture, loadBaseline, loadBeamDatasetPreview, loadBenchmarkArtifact, loadBenchmarkBaseline, loadBenchmarkReportCardProvenance, loadBenchmarkResult, loadCustomBenchmarkFile, loadJudgeCalibrationState, loadLoCoMo10, loadLocalLabManifest, loadLongMemEvalS, loadSealKeyFromEnv, loadSealedQrels, loadSealedRubric, matchEntity, mergeContaminationManifests, openSeal, orchestrateBenchmarkRuns, pairedDeltaConfidenceInterval, parseBenchmarkArtifact, parseCustomBenchmark, parseLocalLabManifest, parseRubricResponse, parseSealedQrels, pickStableQualifiedName, precisionAtK, preflightLocalLabRole, projectFolderFixture, recallAtK, redactBenchmarkResultSecrets, renderBaselineMarkdown, renderBenchmarkResultExport, renderLoComoProfileDeltaMarkdown, renderLoComoRecallDeltaMarkdown, renderMemorySummaryForJudge, renderMemoryViewForAgent, resolveAssistantAgent, resolveAssistantRubricId, resolveAssistantSeeds, resolveAssistantSpotCheckDir, resolveBenchRuntimeProfile, resolveBenchmarkPhaseTimeoutMs, resolveBenchmarkProgressLogging, resolveBenchmarkResultReference, resolveBenchmarkRunCount, resolveLocalLabProfile, resolveLocalLabRole, resolveStructuredJudge, rotateDistractors, rougeL, runAssistantBenchmark, runAssistantMeetingPrepBenchmark, runAssistantMorningBriefBenchmark, runAssistantNextBestActionBenchmark, runAssistantSynthesisBenchmark, runBaseline, runBenchSuite, runBenchmark, runCodingGraphBenchmark, runCustomBenchmarkFile, runExplain, runExtractionAttack, runJudgeCalibration, runMitigatedBaseline, runProceduralAblation, runProceduralAblationCli, runSealedJudge, runSequentialPhases, safeHexEqual, sanitizeLoComoResultReference, saveBaseline, saveBenchmarkBaseline, schemaCompleteness, sealPayload, selectAmaBenchDiagnosticVariants, selectCalibrationSlice, selectFixtureVariant, serializeBenchmarkArtifact, serializeJsonl, serializeSealedQrels, shuffleTasks, timed, verifyRubricDigest, writeBenchmarkArtifact, writeBenchmarkPublishFeed, writeBenchmarkReproManifest, writeBenchmarkResult, writeJudgeCalibrationState, writeLeaderboardArtifactsForResult, zeroScores };
|