@remnic/bench 9.7.15 → 9.8.0

package/dist/index.d.ts

@@ -552,6 +552,15 @@ interface BenchmarkResult {
         tasks: TaskResult[];
         aggregates: AggregateMetrics;
         statistics?: StatisticalReport;
+        /**
+         * Optional per-category aggregate breakdown keyed by a benchmark-defined
+         * category label (e.g. LoCoMo's single_hop/adversarial). Populated by
+         * benchmarks whose tasks carry a `categoryName` detail so per-category
+         * metrics — such as the adversarial-vs-answerable split issue #1878
+         * tracks — are read straight from the artifact instead of hand-computed
+         * from task ids.
+         */
+        categoryAggregates?: Record<string, AggregateMetrics>;
     };
     environment: {
         os: string;
@@ -1748,6 +1757,13 @@ interface CodexCliNativeUsage {
     outputTokens: number;
     reasoningOutputTokens: number;
 }
+interface CodexCreditBudgetConfig {
+    budgetCredits: number;
+    reserveCredits: number;
+    ledgerPath: string;
+    allowSol: boolean;
+    runId?: string;
+}
 interface CodexCreditReceiptScope extends CodexCliNativeUsage {
     calls: number;
     budgetUnits: number;
@@ -1789,6 +1805,13 @@ interface CodexCreditReceipt {
         id: string;
     };
 }
+declare class CodexCreditAccountingError extends Error {
+    constructor(message: string);
+}
+/** The Codex child was confirmed not to have started, so no credits were charged. */
+declare class CodexCreditDispatchError extends Error {
+    constructor(message: string, options?: ErrorOptions);
+}
 declare function reconcileCodexCreditLedger(args: {
     ledgerPath: string;
     priorLedgerSha256: string;
@@ -1801,6 +1824,17 @@ declare function reconcileCodexCreditLedger(args: {
     noInterveningCodexActivityConfirmed?: true;
     affectedRunId: string;
 }): Promise<CodexCreditReconciliationReceipt>;
+declare function resolveCodexCreditBudgetConfig(env?: NodeJS.ProcessEnv, fallbackRunId?: string): CodexCreditBudgetConfig | undefined;
+declare function runWithinCodexCreditBudget<T>(args: {
+    config: CodexCreditBudgetConfig | undefined;
+    model: string;
+    run: () => Promise<{
+        value: T;
+        usage: CodexCliNativeUsage;
+    }>;
+    onUsagePersisted?: (usage: CodexCliNativeUsage) => void;
+}): Promise<T>;
+declare function parseCodexJsonlUsage(output: string): CodexCliNativeUsage | undefined;
 declare function calculateCodexBudgetUnits(model: string, usage: CodexCliNativeUsage): number;
 declare function buildCodexCreditReceipt(ledgerPath: string, runId?: string): Promise<CodexCreditReceipt>;
 
@@ -5747,4 +5781,4 @@ declare function checkCodingGraphRegression(report: CodingGraphBenchReport, base
  */
 declare function buildBaselineFromReport(report: CodingGraphBenchReport, note: string): CodingGraphBaseline;
 
-export { AMA_BENCH_DIAGNOSTIC_VARIANTS, ASSISTANT_AGENT_CONFIG_KEY, ASSISTANT_JUDGE_CONFIG_KEY, ASSISTANT_MEETING_PREP_SCENARIOS, ASSISTANT_MEETING_PREP_SMOKE_SCENARIOS, ASSISTANT_MORNING_BRIEF_SCENARIOS, ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SMOKE_SCENARIOS, ASSISTANT_RUBRIC_DIMENSIONS, ASSISTANT_RUBRIC_ID_KEY, ASSISTANT_SEEDS_CONFIG_KEY, ASSISTANT_SPOT_CHECK_DIR_KEY, ASSISTANT_SYNTHESIS_SCENARIOS, ASSISTANT_SYNTHESIS_SMOKE_SCENARIOS, type AblationConfigOverrides, type AbstentionRetrievalCase, type AggregateMetrics, type AmaBenchDiagnosticAdapterOptions, type AmaBenchDiagnosticAnswererMode, type AmaBenchDiagnosticBreakdown, type AmaBenchDiagnosticMatrixArtifact, type AmaBenchDiagnosticRecallMode, type AmaBenchDiagnosticRunContext, type AmaBenchDiagnosticTaskEvidence, type AmaBenchDiagnosticTaskRow, type AmaBenchDiagnosticVariant, type AmaBenchDiagnosticVariantSummary, type AnthropicProviderConfig, type AssistantAgent, type AssistantMemoryFact, type AssistantMemoryGraph, type AssistantRubricDimension, type AssistantRubricRequest, type AssistantRubricScores, type AssistantRunnerOptions, type AssistantScenario, type AssistantStance, type AttackRecallOptions, type AttackRetrievalHit, type AttackerMode, BENCHMARK_ARTIFACT_SCHEMA_VERSION, BENCHMARK_INTEGRITY_META_SCHEMA, BENCHMARK_REPRO_MANIFEST_FILENAME, BENCHMARK_REPRO_MANIFEST_SCHEMA_VERSION, BENCHMARK_RESULT_SCHEMA, BENCHMARK_SPLIT_TYPES, BUILD_WEEK_EVIDENCE_RECEIPT_SCHEMA_VERSION, BUILD_WEEK_LIMITATIONS, type BaselineRow, type BaselineScenario, type BeamDatasetPreview, type BenchConfig, type BenchJudge, type BenchJudgeResult, type BenchMemoryAdapter, type BenchModelSource, type BenchReasoningEffort, type BenchRecallLineageStatus, type BenchRecallOptions, type BenchRecallSupportAssessment, type BenchRecallSupportRequest, type BenchRecallSupportStatus, type BenchRecallTrace, type BenchRecallTraceCoreCapture, type BenchRecallTraceLcmCandidate, type BenchRecallTraceRange, type BenchRecallTraceSection, type BenchRecallTraceSelection, type BenchRecallWithTraceResult, type BenchResponder, type BenchResponse, type BenchRuntimeProfile, type BenchTier, type BenchmarkArtifact, type BenchmarkArtifactEnvironment, type BenchmarkArtifactHardware, type BenchmarkArtifactJudgeCalibration, type BenchmarkArtifactPerTaskScore, type BenchmarkArtifactSystem, type BenchmarkArtifactTier, type BenchmarkCategory, type BenchmarkDefinition, type BenchmarkIntegrityMeta, type BenchmarkMeta, type BenchmarkMode, type BenchmarkReport, type BenchmarkReproManifest, type BenchmarkReproManifestDataset, type BenchmarkReproManifestFile, type BenchmarkReproManifestResult, type BenchmarkResult, type BenchmarkSplitType, type BenchmarkStatus, type BenchmarkSuiteResult, type BenchmarkTier, type BootstrapKappaOptions, type BootstrapKappaResult, type BuildBenchmarkArtifactInput, type BuildBenchmarkPublishFeedOptions, type BuildBenchmarkReproManifestOptions, type BuildBuildWeekEvidenceReceiptOptions, type BuildWeekEvidenceReceipt, type BuildWeekEvidenceReceiptProvider, type BuildWeekLimitationCode, type BuiltInProvider, CALIBRATION_SLICE_SIZE, CANARY_FIXED_RECALL, CANARY_SCORE_FLOOR, DEFAULT_10K_FIXTURE as CODING_GRAPH_10K_FIXTURE, CODING_GRAPH_BENCH_SCHEMA_VERSION, DEFAULT_TOLERANCE_PERCENT as CODING_GRAPH_DEFAULT_TOLERANCE, MIN_ITERATIONS as CODING_GRAPH_MIN_ITERATIONS, DEFAULT_SMOKE_FIXTURE as CODING_GRAPH_SMOKE_FIXTURE, type CalibrationAnswer, type CalibrationVerdictPair, type CanaryAdapterOptions, type CanaryFloorCheck, type CaptureLoCoMoRetrievalTraceOptions, type ClaudeCliProviderConfig, type CodexCliProviderConfig, type CodexCreditReceipt, type CodexCreditReceiptScope, type CodexCreditReconciliationReceipt, type CodingGraphBaseline, type CodingGraphBenchConfig, type CodingGraphBenchReport, type MachineFingerprint as CodingGraphMachineFingerprint, type CodingGraphMetricKey, type RegressionMetricDetail as CodingGraphRegressionDetail, type RegressionMetricKey as CodingGraphRegressionKey, type RegressionGateResult as CodingGraphRegressionResult, type CohenKappaResult, type ComparisonMetricDelta, type ComparisonResult, type CompletionOpts, type CompletionResult, type ConfidenceInterval, type ContaminationCheckResult, type ContaminationEntry, type ContaminationManifest, type CustomBenchmarkScoring, type CustomBenchmarkSpec, type CustomBenchmarkTask, DEFAULT_ABLATION_BENCHMARK, DEFAULT_ABLATION_BOOTSTRAP_SEED, DEFAULT_ASSISTANT_RUBRIC_ID, DEFAULT_BASELINE_SCENARIOS, DEFAULT_JUDGE_BINARIZATION_THRESHOLD, DEFAULT_KAPPA_BOOTSTRAP_SAMPLES, DEFAULT_KAPPA_CONFIDENCE_LEVEL, DEFAULT_OPENAI_RESPONSES_JUDGE_MODEL, type DatasetSource, type DiagnoseLoComoProfileDeltaOptions, type DiagnoseLoComoRecallDeltaOptions, type DiscoveredModel, EMPTY_CONTAMINATION_MANIFEST, type EffectSizeInterpretation, type EffectSizeSummary, type ExplainResult, type ExtractedEntity, type ExtractedLink, type ExtractedPage, type ExtractionAttackOptions, type ExtractionAttackResult, type ExtractionAttackTarget, type FixtureGenerator, type FixtureOutput, type FixtureVariant, GENERAL_ANSWER_JUDGE_RUBRIC, type GeneratedFile, type GeneratedRepo, type GoldEntity, type GoldEntityType, type GoldGraph, type GoldLink, type GoldPage, type HarnessRng, INTEGRITY_CIPHER_ALGORITHM, INTEGRITY_HASH_ALGORITHM, INTEGRITY_META_FIELDS, type IngestionBenchAdapter, type IngestionLog, JUDGE_CALIBRATION_KAPPA_THRESHOLD, JUDGE_CALIBRATION_PROTOCOL_VERSION, type JudgeCalibrationCheckpointProvenance, type JudgeCalibrationIdentities, type JudgeCalibrationResult, type JudgeCategory, type KappaConfidenceInterval, LOCAL_LAB_PROVIDER_KINDS, LOCOMO_DATASET_FILENAMES, LOCOMO_FULL_TASK_COUNT, LOCOMO_RECALL_DIFF_LINE_LIMIT, LOCOMO_RECALL_EXCERPT_CHARS, LOCOMO_RETRIEVAL_TRACE_BUDGET_VERSION, LOCOMO_RETRIEVAL_TRACE_DELTA_SCHEMA_VERSION, LOCOMO_RETRIEVAL_TRACE_SCHEMA_VERSION, LOCOMO_RETRIEVAL_TRACE_SELECTION_VERSION, LONG_MEM_EVAL_DATASET_FILENAMES, type LeaderboardArtifactWrite, type LettaAdapterConfig, LettaMemCorrectAdapter, type LlmJudge, type LlmProvider, type LoCoMoCategory, type LoCoMoRetrievalMechanism, type LoCoMoRetrievalMechanismSummary, type LoCoMoRetrievalSessionReceipt, type LoCoMoRetrievalStructuralTrace, type LoCoMoRetrievalTaskDelta, type LoCoMoRetrievalTaskReceipt, type LoCoMoRetrievalTraceCoreCaptureReceipt, type LoCoMoRetrievalTraceDeltaReport, type LoCoMoRetrievalTraceProfile, type LoCoMoRetrievalTraceReceipt, type LoCoMoRetrievalTraceSelectionManifest, type LoCoMoRetrievalTraceSelector, type LoCoMoStructuralMultisetDelta, type LoComoCategoryDelta, type LoComoFinalContextRegression, type LoComoMetricDelta, type LoComoProfileArtifactEvidence, type LoComoProfileDeltaReport, type LoComoRawResultEvidence, type LoComoRecallCategoryDelta, type LoComoRecallContextSummary, type LoComoRecallDeltaReport, type LoComoRecallLineDelta, type LoComoRecallLineEvidence, type LoComoRecallMetricDelta, type LoComoRecallResultProvenance, type LoComoRecallTextDigest, type LoComoTaskRegression, type LoadDatasetOptions, type LoadSealedQrelsOptions, type LoadedDataset, type LoadedJudgeCalibrationState, type LocalLabManifest, type LocalLabManifestNotes, type LocalLabPhase, type LocalLabPhaseDescriptor, type LocalLabPhaseExecute, type LocalLabPhaseName, type LocalLabPhaseOutcome, LocalLabPreflightError, type LocalLabPreflightFailure, type LocalLabPreflightInput, type LocalLabPreflightOptions, type LocalLabPreflightResult, type LocalLabPreflightSuccess, type LocalLabProviderKind, type LocalLabRoleConfig, type LocalLlmProviderConfig, MEMCORRECT_CORRECTION_ACCEPTANCE_RUBRIC, MEMCORRECT_CORRECTION_ACCEPTANCE_RUBRIC_VERSION, MEMCORRECT_STALE_HARM_RUBRIC, MEMCORRECT_STALE_HARM_RUBRIC_VERSION, MEMORY_EVAL_DIMENSIONS, MEMORY_EVAL_PUBLIC_LINE, MIN_CALIBRATION_SOURCE_TASKS, MITIGATED_BASELINE_SCENARIOS, type McpArgumentSemantic, type McpBackendErrorCode, type McpBackendResult, type McpBenchMemoryAdapter, type McpConformanceResult, type McpHttpTransportConfig, type McpListedTool, type McpMemCorrectAdapter, type McpMemoryAdapterOptions, McpMemoryBackendError, type McpMemoryToolMapping, type McpMemoryTransportConfig, type McpStdioTransportConfig, type McpToolCallResult, type McpToolClient, type McpToolMappingEntry, type McpToolMappingValue, type McpToolOperation, type Mem0AdapterConfig, Mem0MemCorrectAdapter, type MemCorrectGeneratorOptions, type MemCorrectJudgeRequest, type MemCorrectJudgeResult, type MemCorrectSystemAdapter, type MemoryEvalCategory, type MemoryEvalDimension, type MemoryEvalDimensionId, type MemoryEvalMetric, type MemoryGraph, type MemoryStats, type MemorySystem, type Message, type MetricAggregate, type MicroMetric, MissingCredentialError, type MitigatedBaselineConfig, type MitigatedTargetConfig, type MultipleChoiceQuestion, OPENAI_RESPONSES_JUDGE_RUBRIC_VERSION, OTHER_NAMESPACE_MEMORIES, type OllamaProviderConfig, type OpenAiCompatibleProviderConfig, OpenAiResponsesJudgeError, type OpenAiResponsesJudgeErrorCode, type OpenAiResponsesJudgeTelemetry, OpenAiResponsesProvider, type OpenAiResponsesProviderConfig, type OpenAiResponsesVerdict, type OpenAiResponsesVerdictResult, PROCEDURAL_REAL_SCENARIOS, PROCEDURAL_REAL_SCENARIOS_SMOKE, PUBLISHED_BENCHMARK_ARTIFACT_IDS, type PersonalizationRetrievalCase, type PreflightDiscoveredModel, type ProceduralAblationArtifact, type ProceduralAblationPerCase, type ProceduralAblationScenario, type ProceduralRealScenario, type ProceduralRealScenarioCategory, type ProviderBaseConfig, type ProviderConfig, type ProviderDiscoveryResult, type ProviderFactoryConfig, type PublishSkipReason, type PublishSkipRecord, type PublishedBenchmarkFeed, type PublishedBenchmarkFeedEntry, type PublishedBenchmarkId, REQUIRED_FRONTMATTER_FIELDS, type RecallMetrics, type RecoveredMemory, type RegressionDetail, type RegressionGateResult$1 as RegressionGateResult, type RemnicAdapterOptions, type ReportCardProvenanceContext, type ResolveBenchRuntimeProfileOptions, type ResolvedBenchRuntimeProfile, type ResolvedLocalLabProfile, type ResolvedLocalLabRole, type ResolvedRunBenchmarkOptions, type RotatedChoices, type RunBenchmarkOptions, type RunJudgeCalibrationOptions, type RunProceduralAblationCliArgs, type RunProceduralAblationOptions, type RunSequentialPhasesOptions, SCHEMA_TIER_FIXTURE, SCHEMA_TIER_SMOKE_FIXTURE, SEALED_PROMPT_REGISTRY, SINGLE_FLAG_ABLATION_MATRIX, SYNTHETIC_MEMORIES, type SanitizedDiagnosticProvider, type SavedBaseline, type SchemaTierCorpus, type SchemaTierFixture, type SchemaTierName, type SchemaTierPage, type SchemaTierPageFrontmatter, type SealedArtifact, type SealedJudgeDecision, type SealedJudgeInput, type SealedQrelsArtifact, type SealedQrelsHandle, type SealedRubric, type SearchResult, type SeededMemory, type SeededRng, type SequentialPhaseHooks, type SingleFlagAblationCell, type SingleFlagAblationId, type SpotCheckLogger, type StatisticalReport, type StructuredJudge, StructuredJudgeError, type StructuredJudgeErrorCode, type StructuredJudgeProvider, type StructuredJudgeTelemetry, type StructuredJudgeVerdict, type StructuredJudgeVerdictResult, type StructuredVerdictRequest, type SyntheticEdge, type SyntheticEmailIngestionAdapterOptions, type SyntheticFileIR, type SyntheticRepoConfig, type SyntheticSymbol, type SyntheticTargetOptions, type TaskResult, type TaskTokenUsage, type TemporalRetrievalCase, type ThirdPartyAdapterConfig, type TierDetail, type TimelineEntry, type TokenUsage, type WallMetric, type WriteBenchmarkArtifactResult, type ZepAdapterConfig, ZepMemCorrectAdapter, addContaminationEntry, aggregateTaskScores, answerBenchmarkQuestion, assertCanaryUnderFloor, assertIntegrityMetaPresent, assertPublishableIntegrity, assertSha256Hex, assistantMeetingPrepDefinition, assistantMorningBriefDefinition, assistantNextBestActionDefinition, assistantSynthesisDefinition, backlinkF1, binarizeJudgeScore, bootstrapCohensKappaConfidenceInterval, bootstrapMeanConfidenceInterval, buildAmaBenchDiagnosticMatrixArtifact, buildAmaBenchDiagnosticVariantSummary, buildAmaBenchLeaderboardRows, buildBaselineFromReport, buildBenchmarkArtifact, buildBenchmarkArtifactFilename, buildBenchmarkPublishFeed, buildBenchmarkReproManifest, buildBenchmarkRunSeeds, buildBuildWeekEvidenceReceipt, buildCodexCreditReceipt, buildJudgePayload, buildOracleTrajectoryRecall, buildProviderFreeLoCoMoRetrievalConfig, buildSchemaTierFixture, buildSchemaTierSmokeFixture, calculateCodexBudgetUnits, calendarFixture, canonicalJsonStringify, captureLoCoMoRetrievalTrace, captureMachineFingerprint, chatFixture, checkCodingGraphRegression, checkDatasetContamination, checkRegression, clampScore, cohensD, compareResults, computeCohensKappa, computeSealHash, containsAnswer, createSeededRng$1 as createAdamSeededRng, createAmaBenchDiagnosticAdapter, createAnthropicProvider, createCanaryAdapter, createClaudeCliProvider, createCodexCliProvider, createSeededRng as createCodingGraphSeededRng, createDeterministicSpotCheckLogger, createGatewayResponder, createLightweightAdapter, createLiteLlmProvider, createLocalLlmProvider, createMcpDemoMemCorrectAdapter, createMcpDemoMemoryAdapter, createMcpMemCorrectAdapter, createMcpMemoryAdapter, createMitigatedTarget, createOllamaProvider, createOpenAiCompatibleProvider, createOpenAiResponsesBenchJudge, createOpenAiResponsesProvider, createSeededRandom as createProceduralAblationSeededRandom, createProvider, createProviderBackedAmaBenchRecommendedJudge, createProviderBackedJudge, createProviderBackedResponder, createProviderBackedStructuredJudge, createRemnicAdapter, createResponderFromProvider, createSeededRng$2 as createSeededRng, createSpotCheckFileLogger, createStructuredBenchJudge, createStructuredJudgeFromProvider, createSyntheticEmailIngestionAdapter, createSyntheticTarget, createTimeoutGuardedAdapter, defaultBenchmarkBaselineDir, defaultBenchmarkPublishPath, deleteBenchmarkResults, diagnoseLoCoMoRetrievalTraceDelta, diagnoseLoComoProfileDelta, diagnoseLoComoRecallDelta, discoverAllProviders, discoveryEndpointFor, emailFixture, entityRecall, exactMatch, extractMetrics as extractCodingGraphMetrics, extractMarkdownSectionsByTitle, f1Score, fixtureToAblationScenarios, formatHandoffNote, formatMissingDatasetError, generateReport, generateSyntheticRepo, getAblationCell, getBenchmark, getBenchmarkLowerIsBetter, getGitSha, getMemoryEvalDimension, getProviderBackedJudgePromptIdentity, getRemnicVersion, hashBenchmarkArtifact, hashBytes, hashCanonicalJson, hashOrderedQuestionIds, hashString, integrityMetaIsComplete, interpretEffectSize, isAmaBenchUnknownLikeAnswer, isContaminationEntry, isContaminationManifest, isSealedQrelsArtifact, isSha256Hex, isStructuredJudgeProvider, judgeMemCorrectCorrectionAcceptance, judgeMemCorrectStaleMemoryHarm, linkMatches, listBenchmarkBaselines, listBenchmarkResults, listBenchmarks, listMemoryEvalBenchmarkIds, listMemoryEvalDimensions, llmJudgeScore, llmJudgeScoreDetailed, loadAblationFixture, loadBaseline, loadBeamDatasetPreview, loadBenchmarkArtifact, loadBenchmarkBaseline, loadBenchmarkReportCardProvenance, loadBenchmarkResult, loadCustomBenchmarkFile, loadJudgeCalibrationState, loadLoCoMo10, loadLocalLabManifest, loadLongMemEvalS, loadSealKeyFromEnv, loadSealedQrels, loadSealedRubric, matchEntity, mergeContaminationManifests, openSeal, orchestrateBenchmarkRuns, pairedDeltaConfidenceInterval, parseBenchmarkArtifact, parseCustomBenchmark, parseLocalLabManifest, parseRubricResponse, parseSealedQrels, pickStableQualifiedName, precisionAtK, preflightLoCoMoRetrievalTraceCapture, preflightLocalLabRole, projectFolderFixture, recallAtK, reconcileCodexCreditLedger, redactBenchmarkResultSecrets, renderBaselineMarkdown, renderBenchmarkResultExport, renderLoComoProfileDeltaMarkdown, renderLoComoRecallDeltaMarkdown, renderMemorySummaryForJudge, renderMemoryViewForAgent, resolveAssistantAgent, resolveAssistantRubricId, resolveAssistantSeeds, resolveAssistantSpotCheckDir, resolveBenchRuntimeProfile, resolveBenchmarkPhaseTimeoutMs, resolveBenchmarkProgressLogging, resolveBenchmarkResultReference, resolveBenchmarkRunCount, resolveLocalLabJudgeProviderConfig, resolveLocalLabProfile, resolveLocalLabRole, resolveStructuredJudge, rotateDistractors, rougeL, runAssistantBenchmark, runAssistantMeetingPrepBenchmark, runAssistantMorningBriefBenchmark, runAssistantNextBestActionBenchmark, runAssistantSynthesisBenchmark, runBaseline, runBenchSuite, runBenchmark, runCodingGraphBenchmark, runCustomBenchmarkFile, runExplain, runExtractionAttack, runJudgeCalibration, runMitigatedBaseline, runProceduralAblation, runProceduralAblationCli, runSealedJudge, runSequentialPhases, safeHexEqual, sanitizeLoComoResultReference, saveBaseline, saveBenchmarkBaseline, schemaCompleteness, sealPayload, selectAmaBenchDiagnosticVariants, selectCalibrationSlice, selectFixtureVariant, serializeBenchmarkArtifact, serializeBuildWeekEvidenceReceipt, serializeJsonl, serializeLoCoMoRetrievalTraceDelta, serializeLoCoMoRetrievalTraceReceipt, serializeSealedQrels, shuffleTasks, timed, verifyRubricDigest, writeBenchmarkArtifact, writeBenchmarkPublishFeed, writeBenchmarkReproManifest, writeBenchmarkResult, writeBuildWeekEvidenceReceipt, writeJudgeCalibrationState, writeLeaderboardArtifactsForResult, zeroScores };
+export { AMA_BENCH_DIAGNOSTIC_VARIANTS, ASSISTANT_AGENT_CONFIG_KEY, ASSISTANT_JUDGE_CONFIG_KEY, ASSISTANT_MEETING_PREP_SCENARIOS, ASSISTANT_MEETING_PREP_SMOKE_SCENARIOS, ASSISTANT_MORNING_BRIEF_SCENARIOS, ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SMOKE_SCENARIOS, ASSISTANT_RUBRIC_DIMENSIONS, ASSISTANT_RUBRIC_ID_KEY, ASSISTANT_SEEDS_CONFIG_KEY, ASSISTANT_SPOT_CHECK_DIR_KEY, ASSISTANT_SYNTHESIS_SCENARIOS, ASSISTANT_SYNTHESIS_SMOKE_SCENARIOS, type AblationConfigOverrides, type AbstentionRetrievalCase, type AggregateMetrics, type AmaBenchDiagnosticAdapterOptions, type AmaBenchDiagnosticAnswererMode, type AmaBenchDiagnosticBreakdown, type AmaBenchDiagnosticMatrixArtifact, type AmaBenchDiagnosticRecallMode, type AmaBenchDiagnosticRunContext, type AmaBenchDiagnosticTaskEvidence, type AmaBenchDiagnosticTaskRow, type AmaBenchDiagnosticVariant, type AmaBenchDiagnosticVariantSummary, type AnthropicProviderConfig, type AssistantAgent, type AssistantMemoryFact, type AssistantMemoryGraph, type AssistantRubricDimension, type AssistantRubricRequest, type AssistantRubricScores, type AssistantRunnerOptions, type AssistantScenario, type AssistantStance, type AttackRecallOptions, type AttackRetrievalHit, type AttackerMode, BENCHMARK_ARTIFACT_SCHEMA_VERSION, BENCHMARK_INTEGRITY_META_SCHEMA, BENCHMARK_REPRO_MANIFEST_FILENAME, BENCHMARK_REPRO_MANIFEST_SCHEMA_VERSION, BENCHMARK_RESULT_SCHEMA, BENCHMARK_SPLIT_TYPES, BUILD_WEEK_EVIDENCE_RECEIPT_SCHEMA_VERSION, BUILD_WEEK_LIMITATIONS, type BaselineRow, type BaselineScenario, type BeamDatasetPreview, type BenchConfig, type BenchJudge, type BenchJudgeResult, type BenchMemoryAdapter, type BenchModelSource, type BenchReasoningEffort, type BenchRecallLineageStatus, type BenchRecallOptions, type BenchRecallSupportAssessment, type BenchRecallSupportRequest, type BenchRecallSupportStatus, type BenchRecallTrace, type BenchRecallTraceCoreCapture, type BenchRecallTraceLcmCandidate, type BenchRecallTraceRange, type BenchRecallTraceSection, type BenchRecallTraceSelection, type BenchRecallWithTraceResult, type BenchResponder, type BenchResponse, type BenchRuntimeProfile, type BenchTier, type BenchmarkArtifact, type BenchmarkArtifactEnvironment, type BenchmarkArtifactHardware, type BenchmarkArtifactJudgeCalibration, type BenchmarkArtifactPerTaskScore, type BenchmarkArtifactSystem, type BenchmarkArtifactTier, type BenchmarkCategory, type BenchmarkDefinition, type BenchmarkIntegrityMeta, type BenchmarkMeta, type BenchmarkMode, type BenchmarkReport, type BenchmarkReproManifest, type BenchmarkReproManifestDataset, type BenchmarkReproManifestFile, type BenchmarkReproManifestResult, type BenchmarkResult, type BenchmarkSplitType, type BenchmarkStatus, type BenchmarkSuiteResult, type BenchmarkTier, type BootstrapKappaOptions, type BootstrapKappaResult, type BuildBenchmarkArtifactInput, type BuildBenchmarkPublishFeedOptions, type BuildBenchmarkReproManifestOptions, type BuildBuildWeekEvidenceReceiptOptions, type BuildWeekEvidenceReceipt, type BuildWeekEvidenceReceiptProvider, type BuildWeekLimitationCode, type BuiltInProvider, CALIBRATION_SLICE_SIZE, CANARY_FIXED_RECALL, CANARY_SCORE_FLOOR, DEFAULT_10K_FIXTURE as CODING_GRAPH_10K_FIXTURE, CODING_GRAPH_BENCH_SCHEMA_VERSION, DEFAULT_TOLERANCE_PERCENT as CODING_GRAPH_DEFAULT_TOLERANCE, MIN_ITERATIONS as CODING_GRAPH_MIN_ITERATIONS, DEFAULT_SMOKE_FIXTURE as CODING_GRAPH_SMOKE_FIXTURE, type CalibrationAnswer, type CalibrationVerdictPair, type CanaryAdapterOptions, type CanaryFloorCheck, type CaptureLoCoMoRetrievalTraceOptions, type ClaudeCliProviderConfig, type CodexCliNativeUsage, type CodexCliProviderConfig, CodexCreditAccountingError, type CodexCreditBudgetConfig, CodexCreditDispatchError, type CodexCreditReceipt, type CodexCreditReceiptScope, type CodexCreditReconciliationReceipt, type CodingGraphBaseline, type CodingGraphBenchConfig, type CodingGraphBenchReport, type MachineFingerprint as CodingGraphMachineFingerprint, type CodingGraphMetricKey, type RegressionMetricDetail as CodingGraphRegressionDetail, type RegressionMetricKey as CodingGraphRegressionKey, type RegressionGateResult as CodingGraphRegressionResult, type CohenKappaResult, type ComparisonMetricDelta, type ComparisonResult, type CompletionOpts, type CompletionResult, type ConfidenceInterval, type ContaminationCheckResult, type ContaminationEntry, type ContaminationManifest, type CustomBenchmarkScoring, type CustomBenchmarkSpec, type CustomBenchmarkTask, DEFAULT_ABLATION_BENCHMARK, DEFAULT_ABLATION_BOOTSTRAP_SEED, DEFAULT_ASSISTANT_RUBRIC_ID, DEFAULT_BASELINE_SCENARIOS, DEFAULT_JUDGE_BINARIZATION_THRESHOLD, DEFAULT_KAPPA_BOOTSTRAP_SAMPLES, DEFAULT_KAPPA_CONFIDENCE_LEVEL, DEFAULT_OPENAI_RESPONSES_JUDGE_MODEL, type DatasetSource, type DiagnoseLoComoProfileDeltaOptions, type DiagnoseLoComoRecallDeltaOptions, type DiscoveredModel, EMPTY_CONTAMINATION_MANIFEST, type EffectSizeInterpretation, type EffectSizeSummary, type ExplainResult, type ExtractedEntity, type ExtractedLink, type ExtractedPage, type ExtractionAttackOptions, type ExtractionAttackResult, type ExtractionAttackTarget, type FixtureGenerator, type FixtureOutput, type FixtureVariant, GENERAL_ANSWER_JUDGE_RUBRIC, type GeneratedFile, type GeneratedRepo, type GoldEntity, type GoldEntityType, type GoldGraph, type GoldLink, type GoldPage, type HarnessRng, INTEGRITY_CIPHER_ALGORITHM, INTEGRITY_HASH_ALGORITHM, INTEGRITY_META_FIELDS, type IngestionBenchAdapter, type IngestionLog, JUDGE_CALIBRATION_KAPPA_THRESHOLD, JUDGE_CALIBRATION_PROTOCOL_VERSION, type JudgeCalibrationCheckpointProvenance, type JudgeCalibrationIdentities, type JudgeCalibrationResult, type JudgeCategory, type KappaConfidenceInterval, LOCAL_LAB_PROVIDER_KINDS, LOCOMO_DATASET_FILENAMES, LOCOMO_FULL_TASK_COUNT, LOCOMO_RECALL_DIFF_LINE_LIMIT, LOCOMO_RECALL_EXCERPT_CHARS, LOCOMO_RETRIEVAL_TRACE_BUDGET_VERSION, LOCOMO_RETRIEVAL_TRACE_DELTA_SCHEMA_VERSION, LOCOMO_RETRIEVAL_TRACE_SCHEMA_VERSION, LOCOMO_RETRIEVAL_TRACE_SELECTION_VERSION, LONG_MEM_EVAL_DATASET_FILENAMES, type LeaderboardArtifactWrite, type LettaAdapterConfig, LettaMemCorrectAdapter, type LlmJudge, type LlmProvider, type LoCoMoCategory, type LoCoMoRetrievalMechanism, type LoCoMoRetrievalMechanismSummary, type LoCoMoRetrievalSessionReceipt, type LoCoMoRetrievalStructuralTrace, type LoCoMoRetrievalTaskDelta, type LoCoMoRetrievalTaskReceipt, type LoCoMoRetrievalTraceCoreCaptureReceipt, type LoCoMoRetrievalTraceDeltaReport, type LoCoMoRetrievalTraceProfile, type LoCoMoRetrievalTraceReceipt, type LoCoMoRetrievalTraceSelectionManifest, type LoCoMoRetrievalTraceSelector, type LoCoMoStructuralMultisetDelta, type LoComoCategoryDelta, type LoComoFinalContextRegression, type LoComoMetricDelta, type LoComoProfileArtifactEvidence, type LoComoProfileDeltaReport, type LoComoRawResultEvidence, type LoComoRecallCategoryDelta, type LoComoRecallContextSummary, type LoComoRecallDeltaReport, type LoComoRecallLineDelta, type LoComoRecallLineEvidence, type LoComoRecallMetricDelta, type LoComoRecallResultProvenance, type LoComoRecallTextDigest, type LoComoTaskRegression, type LoadDatasetOptions, type LoadSealedQrelsOptions, type LoadedDataset, type LoadedJudgeCalibrationState, type LocalLabManifest, type LocalLabManifestNotes, type LocalLabPhase, type LocalLabPhaseDescriptor, type LocalLabPhaseExecute, type LocalLabPhaseName, type LocalLabPhaseOutcome, LocalLabPreflightError, type LocalLabPreflightFailure, type LocalLabPreflightInput, type LocalLabPreflightOptions, type LocalLabPreflightResult, type LocalLabPreflightSuccess, type LocalLabProviderKind, type LocalLabRoleConfig, type LocalLlmProviderConfig, MEMCORRECT_CORRECTION_ACCEPTANCE_RUBRIC, MEMCORRECT_CORRECTION_ACCEPTANCE_RUBRIC_VERSION, MEMCORRECT_STALE_HARM_RUBRIC, MEMCORRECT_STALE_HARM_RUBRIC_VERSION, MEMORY_EVAL_DIMENSIONS, MEMORY_EVAL_PUBLIC_LINE, MIN_CALIBRATION_SOURCE_TASKS, MITIGATED_BASELINE_SCENARIOS, type McpArgumentSemantic, type McpBackendErrorCode, type McpBackendResult, type McpBenchMemoryAdapter, type McpConformanceResult, type McpHttpTransportConfig, type McpListedTool, type McpMemCorrectAdapter, type McpMemoryAdapterOptions, McpMemoryBackendError, type McpMemoryToolMapping, type McpMemoryTransportConfig, type McpStdioTransportConfig, type McpToolCallResult, type McpToolClient, type McpToolMappingEntry, type McpToolMappingValue, type McpToolOperation, type Mem0AdapterConfig, Mem0MemCorrectAdapter, type MemCorrectGeneratorOptions, type MemCorrectJudgeRequest, type MemCorrectJudgeResult, type MemCorrectSystemAdapter, type MemoryEvalCategory, type MemoryEvalDimension, type MemoryEvalDimensionId, type MemoryEvalMetric, type MemoryGraph, type MemoryStats, type MemorySystem, type Message, type MetricAggregate, type MicroMetric, MissingCredentialError, type MitigatedBaselineConfig, type MitigatedTargetConfig, type MultipleChoiceQuestion, OPENAI_RESPONSES_JUDGE_RUBRIC_VERSION, OTHER_NAMESPACE_MEMORIES, type OllamaProviderConfig, type OpenAiCompatibleProviderConfig, OpenAiResponsesJudgeError, type OpenAiResponsesJudgeErrorCode, type OpenAiResponsesJudgeTelemetry, OpenAiResponsesProvider, type OpenAiResponsesProviderConfig, type OpenAiResponsesVerdict, type OpenAiResponsesVerdictResult, PROCEDURAL_REAL_SCENARIOS, PROCEDURAL_REAL_SCENARIOS_SMOKE, PUBLISHED_BENCHMARK_ARTIFACT_IDS, type PersonalizationRetrievalCase, type PreflightDiscoveredModel, type ProceduralAblationArtifact, type ProceduralAblationPerCase, type ProceduralAblationScenario, type ProceduralRealScenario, type ProceduralRealScenarioCategory, type ProviderBaseConfig, type ProviderConfig, type ProviderDiscoveryResult, type ProviderFactoryConfig, type PublishSkipReason, type PublishSkipRecord, type PublishedBenchmarkFeed, type PublishedBenchmarkFeedEntry, type PublishedBenchmarkId, REQUIRED_FRONTMATTER_FIELDS, type RecallMetrics, type RecoveredMemory, type RegressionDetail, type RegressionGateResult$1 as RegressionGateResult, type RemnicAdapterOptions, type ReportCardProvenanceContext, type ResolveBenchRuntimeProfileOptions, type ResolvedBenchRuntimeProfile, type ResolvedLocalLabProfile, type ResolvedLocalLabRole, type ResolvedRunBenchmarkOptions, type RotatedChoices, type RunBenchmarkOptions, type RunJudgeCalibrationOptions, type RunProceduralAblationCliArgs, type RunProceduralAblationOptions, type RunSequentialPhasesOptions, SCHEMA_TIER_FIXTURE, SCHEMA_TIER_SMOKE_FIXTURE, SEALED_PROMPT_REGISTRY, SINGLE_FLAG_ABLATION_MATRIX, SYNTHETIC_MEMORIES, type SanitizedDiagnosticProvider, type SavedBaseline, type SchemaTierCorpus, type SchemaTierFixture, type SchemaTierName, type SchemaTierPage, type SchemaTierPageFrontmatter, type SealedArtifact, type SealedJudgeDecision, type SealedJudgeInput, type SealedQrelsArtifact, type SealedQrelsHandle, type SealedRubric, type SearchResult, type SeededMemory, type SeededRng, type SequentialPhaseHooks, type SingleFlagAblationCell, type SingleFlagAblationId, type SpotCheckLogger, type StatisticalReport, type StructuredJudge, StructuredJudgeError, type StructuredJudgeErrorCode, type StructuredJudgeProvider, type StructuredJudgeTelemetry, type StructuredJudgeVerdict, type StructuredJudgeVerdictResult, type StructuredVerdictRequest, type SyntheticEdge, type SyntheticEmailIngestionAdapterOptions, type SyntheticFileIR, type SyntheticRepoConfig, type SyntheticSymbol, type SyntheticTargetOptions, type TaskResult, type TaskTokenUsage, type TemporalRetrievalCase, type ThirdPartyAdapterConfig, type TierDetail, type TimelineEntry, type TokenUsage, type WallMetric, type WriteBenchmarkArtifactResult, type ZepAdapterConfig, ZepMemCorrectAdapter, addContaminationEntry, aggregateTaskScores, answerBenchmarkQuestion, assertCanaryUnderFloor, assertIntegrityMetaPresent, assertPublishableIntegrity, assertSha256Hex, assistantMeetingPrepDefinition, assistantMorningBriefDefinition, assistantNextBestActionDefinition, assistantSynthesisDefinition, backlinkF1, binarizeJudgeScore, bootstrapCohensKappaConfidenceInterval, bootstrapMeanConfidenceInterval, buildAmaBenchDiagnosticMatrixArtifact, buildAmaBenchDiagnosticVariantSummary, buildAmaBenchLeaderboardRows, buildBaselineFromReport, buildBenchmarkArtifact, buildBenchmarkArtifactFilename, buildBenchmarkPublishFeed, buildBenchmarkReproManifest, buildBenchmarkRunSeeds, buildBuildWeekEvidenceReceipt, buildCodexCreditReceipt, buildJudgePayload, buildOracleTrajectoryRecall, buildProviderFreeLoCoMoRetrievalConfig, buildSchemaTierFixture, buildSchemaTierSmokeFixture, calculateCodexBudgetUnits, calendarFixture, canonicalJsonStringify, captureLoCoMoRetrievalTrace, captureMachineFingerprint, chatFixture, checkCodingGraphRegression, checkDatasetContamination, checkRegression, clampScore, cohensD, compareResults, computeCohensKappa, computeSealHash, containsAnswer, createSeededRng$1 as createAdamSeededRng, createAmaBenchDiagnosticAdapter, createAnthropicProvider, createCanaryAdapter, createClaudeCliProvider, createCodexCliProvider, createSeededRng as createCodingGraphSeededRng, createDeterministicSpotCheckLogger, createGatewayResponder, createLightweightAdapter, createLiteLlmProvider, createLocalLlmProvider, createMcpDemoMemCorrectAdapter, createMcpDemoMemoryAdapter, createMcpMemCorrectAdapter, createMcpMemoryAdapter, createMitigatedTarget, createOllamaProvider, createOpenAiCompatibleProvider, createOpenAiResponsesBenchJudge, createOpenAiResponsesProvider, createSeededRandom as createProceduralAblationSeededRandom, createProvider, createProviderBackedAmaBenchRecommendedJudge, createProviderBackedJudge, createProviderBackedResponder, createProviderBackedStructuredJudge, createRemnicAdapter, createResponderFromProvider, createSeededRng$2 as createSeededRng, createSpotCheckFileLogger, createStructuredBenchJudge, createStructuredJudgeFromProvider, createSyntheticEmailIngestionAdapter, createSyntheticTarget, createTimeoutGuardedAdapter, defaultBenchmarkBaselineDir, defaultBenchmarkPublishPath, deleteBenchmarkResults, diagnoseLoCoMoRetrievalTraceDelta, diagnoseLoComoProfileDelta, diagnoseLoComoRecallDelta, discoverAllProviders, discoveryEndpointFor, emailFixture, entityRecall, exactMatch, extractMetrics as extractCodingGraphMetrics, extractMarkdownSectionsByTitle, f1Score, fixtureToAblationScenarios, formatHandoffNote, formatMissingDatasetError, generateReport, generateSyntheticRepo, getAblationCell, getBenchmark, getBenchmarkLowerIsBetter, getGitSha, getMemoryEvalDimension, getProviderBackedJudgePromptIdentity, getRemnicVersion, hashBenchmarkArtifact, hashBytes, hashCanonicalJson, hashOrderedQuestionIds, hashString, integrityMetaIsComplete, interpretEffectSize, isAmaBenchUnknownLikeAnswer, isContaminationEntry, isContaminationManifest, isSealedQrelsArtifact, isSha256Hex, isStructuredJudgeProvider, judgeMemCorrectCorrectionAcceptance, judgeMemCorrectStaleMemoryHarm, linkMatches, listBenchmarkBaselines, listBenchmarkResults, listBenchmarks, listMemoryEvalBenchmarkIds, listMemoryEvalDimensions, llmJudgeScore, llmJudgeScoreDetailed, loadAblationFixture, loadBaseline, loadBeamDatasetPreview, loadBenchmarkArtifact, loadBenchmarkBaseline, loadBenchmarkReportCardProvenance, loadBenchmarkResult, loadCustomBenchmarkFile, loadJudgeCalibrationState, loadLoCoMo10, loadLocalLabManifest, loadLongMemEvalS, loadSealKeyFromEnv, loadSealedQrels, loadSealedRubric, matchEntity, mergeContaminationManifests, openSeal, orchestrateBenchmarkRuns, pairedDeltaConfidenceInterval, parseBenchmarkArtifact, parseCodexJsonlUsage, parseCustomBenchmark, parseLocalLabManifest, parseRubricResponse, parseSealedQrels, pickStableQualifiedName, precisionAtK, preflightLoCoMoRetrievalTraceCapture, preflightLocalLabRole, projectFolderFixture, recallAtK, reconcileCodexCreditLedger, redactBenchmarkResultSecrets, renderBaselineMarkdown, renderBenchmarkResultExport, renderLoComoProfileDeltaMarkdown, renderLoComoRecallDeltaMarkdown, renderMemorySummaryForJudge, renderMemoryViewForAgent, resolveAssistantAgent, resolveAssistantRubricId, resolveAssistantSeeds, resolveAssistantSpotCheckDir, resolveBenchRuntimeProfile, resolveBenchmarkPhaseTimeoutMs, resolveBenchmarkProgressLogging, resolveBenchmarkResultReference, resolveBenchmarkRunCount, resolveCodexCreditBudgetConfig, resolveLocalLabJudgeProviderConfig, resolveLocalLabProfile, resolveLocalLabRole, resolveStructuredJudge, rotateDistractors, rougeL, runAssistantBenchmark, runAssistantMeetingPrepBenchmark, runAssistantMorningBriefBenchmark, runAssistantNextBestActionBenchmark, runAssistantSynthesisBenchmark, runBaseline, runBenchSuite, runBenchmark, runCodingGraphBenchmark, runCustomBenchmarkFile, runExplain, runExtractionAttack, runJudgeCalibration, runMitigatedBaseline, runProceduralAblation, runProceduralAblationCli, runSealedJudge, runSequentialPhases, runWithinCodexCreditBudget, safeHexEqual, sanitizeLoComoResultReference, saveBaseline, saveBenchmarkBaseline, schemaCompleteness, sealPayload, selectAmaBenchDiagnosticVariants, selectCalibrationSlice, selectFixtureVariant, serializeBenchmarkArtifact, serializeBuildWeekEvidenceReceipt, serializeJsonl, serializeLoCoMoRetrievalTraceDelta, serializeLoCoMoRetrievalTraceReceipt, serializeSealedQrels, shuffleTasks, timed, verifyRubricDigest, writeBenchmarkArtifact, writeBenchmarkPublishFeed, writeBenchmarkReproManifest, writeBenchmarkResult, writeBuildWeekEvidenceReceipt, writeJudgeCalibrationState, writeLeaderboardArtifactsForResult, zeroScores };

package/dist/index.js

@@ -5585,6 +5585,43 @@ import {
   serializeEntityFile,
   StorageManager
 } from "@remnic/core";
+
+// src/adapters/with-bench-core-memory-source.ts
+import { composeMemoryEnvelope } from "@remnic/core";
+async function withBenchCoreMemorySource(orchestrator, source, task) {
+  const storage = orchestrator.storage;
+  const originalWriteSealedMemory = storage.writeSealedMemory;
+  const boundWriteSealedMemory = originalWriteSealedMemory.bind(storage);
+  storage.writeSealedMemory = async (envelope, extras) => {
+    const requestedSource = envelope.source;
+    const sourcedEnvelope = composeMemoryEnvelope(
+      {
+        content: envelope.content,
+        category: envelope.category,
+        tags: [...envelope.tags],
+        structuredAttributes: envelope.rawStructuredAttributes ? { ...envelope.rawStructuredAttributes } : void 0,
+        entityRef: envelope.entityRef,
+        confidence: envelope.confidence,
+        ttl: envelope.ttl,
+        validAt: envelope.validAt,
+        sourceConnector: envelope.sourceConnector,
+        sourceReason: envelope.sourceReason
+      },
+      {
+        source: !requestedSource || requestedSource === "extraction" ? source : requestedSource,
+        now: () => new Date(envelope.composedAt)
+      }
+    );
+    return boundWriteSealedMemory(sourcedEnvelope, extras);
+  };
+  try {
+    await task();
+  } finally {
+    storage.writeSealedMemory = originalWriteSealedMemory;
+  }
+}
+
+// src/adapters/remnic-adapter.ts
 import {
   lcmEvidenceIdentity
 } from "@remnic/core/lcm";
@@ -6704,25 +6741,6 @@ async function rememberNewBenchCoreMemories(orchestrator, sessionMemoryIds, sess
     sessionMemoryIds.set(sessionId, remembered);
   }
 }
-async function withBenchCoreMemorySource(orchestrator, sessionId, task) {
-  const storage = orchestrator.storage;
-  const originalWriteMemory = storage.writeMemory;
-  const writeMemory = originalWriteMemory.bind(orchestrator.storage);
-  const source = benchCoreMemorySource(sessionId);
-  storage.writeMemory = async (...args) => {
-    const [category, content, options] = args;
-    const requestedSource = options?.source;
-    return writeMemory(category, content, {
-      ...options ?? {},
-      source: !requestedSource || requestedSource === "extraction" ? source : requestedSource
-    });
-  };
-  try {
-    await task();
-  } finally {
-    storage.writeMemory = originalWriteMemory;
-  }
-}
 async function clearBenchCoreSessionMemories(orchestrator, sessionId, memoryIds, coldCollection) {
   const memories = [
     ...await orchestrator.storage.readAllMemories(),
@@ -7255,7 +7273,7 @@ function createAdapterFactory(mode) {
           try {
             replayIngestion = withBenchCoreMemorySource(
               replayOrchestrator,
-              sessionId,
+              benchCoreMemorySource(sessionId),
               () => withBenchEntityStructuredFactCapture(
                 replayOrchestrator,
                 sessionId,
@@ -21490,6 +21508,27 @@ function formatMissingDatasetError(benchmark, datasetDir, filenames, errors) {
 
 // src/benchmarks/published/harness.ts
 import { randomUUID as randomUUID5 } from "crypto";
+
+// src/benchmarks/published/category-aggregates.ts
+function computeCategoryAggregates(tasks) {
+  const scoresByCategory = /* @__PURE__ */ new Map();
+  for (const task of tasks) {
+    const categoryName = task.details?.categoryName;
+    if (typeof categoryName !== "string" || categoryName.trim().length === 0) {
+      continue;
+    }
+    const bucket = scoresByCategory.get(categoryName);
+    if (bucket) {
+      bucket.push(task.scores);
+    } else {
+      scoresByCategory.set(categoryName, [task.scores]);
+    }
+  }
+  const sortedEntries = [...scoresByCategory.entries()].sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0);
+  return Object.fromEntries(sortedEntries.map(([categoryName, scores]) => [categoryName, aggregateTaskScores(scores)]));
+}
+
+// src/benchmarks/published/harness.ts
 async function* toAsyncIterable(iter) {
   for await (const value of iter) {
     yield value;
@@ -21614,6 +21653,11 @@ async function executeTrialWithFailure(ctx, trial, planIndex, answerSupportGate)
       latencyMs: 0,
       tokens: { input: 0, output: 0 },
       details: {
+        // Preserve the trial's category so a failed trial is still attributed
+        // to its per-category bucket (computeCategoryAggregates), keeping the
+        // per-category breakdown consistent with the overall aggregates that
+        // already count this failure row (issue #1878).
+        ...typeof trial.extraDetails?.categoryName === "string" ? { categoryName: trial.extraDetails.categoryName } : {},
         // `error` is retained for compatibility with existing diagnostics.
         // The structured marker is the authoritative run-status signal; an
         // arbitrary benchmark-owned `extraDetails.error` must not make a
@@ -21981,6 +22025,7 @@ async function buildBenchmarkResult(ctx, tasks, executionProvenance) {
     }];
   });
   const failureReason = failedTasks.length > 0 ? `trial_execution_failure: ${failedTasks.length}/${tasks.length} scored trial(s) failed (${failedTasks.slice(0, 3).map((failure) => `${failure.taskId}: ${failure.message.slice(0, 240)}`).join("; ")}${failedTasks.length > 3 ? `; and ${failedTasks.length - 3} more` : ""})` : void 0;
+  const categoryAggregates = computeCategoryAggregates(tasks);
   return {
     meta: {
       id: randomUUID5(),
@@ -22012,7 +22057,11 @@ async function buildBenchmarkResult(ctx, tasks, executionProvenance) {
     },
     results: {
       tasks,
-      aggregates: aggregateTaskScores(tasks.map((task) => task.scores))
+      aggregates: aggregateTaskScores(tasks.map((task) => task.scores)),
+      // Per-category breakdown for benchmarks that stamp a `categoryName`
+      // detail (LoCoMo). Omitted when empty so other benchmarks' output shape
+      // is unchanged (issue #1878).
+      ...Object.keys(categoryAggregates).length > 0 ? { categoryAggregates } : {}
     },
     environment: {
       os: process.platform,
@@ -32505,6 +32554,7 @@ import {
   buildProcedureRecallSection,
   buildProcedureMarkdownBody
 } from "@remnic/core";
+import { composeMemoryEnvelope as composeMemoryEnvelope2 } from "@remnic/core/write-envelope";
 
 // src/benchmarks/remnic/procedural-recall/fixture.ts
 var PROCEDURAL_RECALL_INTENT_FIXTURE = [
@@ -32632,12 +32682,18 @@ async function runProceduralRecallBenchmark(options) {
       const storage = new StorageManager3(dir);
       await storage.ensureDirectories();
       const body = buildProcedureMarkdownBody(sample.procedureSteps);
-      await storage.writeMemory(
-        "procedure",
-        `${sample.procedurePreamble}
+      await storage.writeSealedMemory(
+        composeMemoryEnvelope2(
+          {
+            content: `${sample.procedurePreamble}
 
 ${body}`,
-        { source: "bench", tags: sample.procedureTags }
+            category: "procedure",
+            tags: sample.procedureTags
+          },
+          { source: "bench" }
+        ),
+        {}
       );
       const config = parseConfig3({
         memoryDir: dir,
@@ -43918,6 +43974,7 @@ import {
   buildProcedureRecallSection as buildProcedureRecallSection2,
   buildProcedureMarkdownBody as buildProcedureMarkdownBody2
 } from "@remnic/core";
+import { composeMemoryEnvelope as composeMemoryEnvelope3 } from "@remnic/core/write-envelope";
 function scoreCase4(expectMatch, observedMatch) {
   return observedMatch === expectMatch ? 1 : 0;
 }
@@ -43948,12 +44005,18 @@ async function runSide(scenarios, proceduralEnabled) {
       const storage = new StorageManager4(dir);
       await storage.ensureDirectories();
       const body = buildProcedureMarkdownBody2(scenario.procedureSteps);
-      await storage.writeMemory(
-        "procedure",
-        `${scenario.procedurePreamble}
+      await storage.writeSealedMemory(
+        composeMemoryEnvelope3(
+          {
+            content: `${scenario.procedurePreamble}
 
 ${body}`,
-        { source: "bench", tags: scenario.procedureTags }
+            category: "procedure",
+            tags: scenario.procedureTags
+          },
+          { source: "bench" }
+        ),
+        {}
       );
       const config = parseConfig5({
         memoryDir: dir,
@@ -45827,6 +45890,8 @@ export {
   DEFAULT_TOLERANCE_PERCENT as CODING_GRAPH_DEFAULT_TOLERANCE,
   MIN_ITERATIONS as CODING_GRAPH_MIN_ITERATIONS,
   DEFAULT_SMOKE_FIXTURE as CODING_GRAPH_SMOKE_FIXTURE,
+  CodexCreditAccountingError,
+  CodexCreditDispatchError,
   DEFAULT_ABLATION_BENCHMARK,
   DEFAULT_ABLATION_BOOTSTRAP_SEED,
   DEFAULT_ASSISTANT_RUBRIC_ID,
@@ -46032,6 +46097,7 @@ export {
   orchestrateBenchmarkRuns,
   pairedDeltaConfidenceInterval,
   parseBenchmarkArtifact,
+  parseCodexJsonlUsage,
   parseCustomBenchmark,
   parseLocalLabManifest,
   parseRubricResponse,
@@ -46059,6 +46125,7 @@ export {
   resolveBenchmarkProgressLogging,
   resolveBenchmarkResultReference,
   resolveBenchmarkRunCount,
+  resolveCodexCreditBudgetConfig,
   resolveLocalLabJudgeProviderConfig,
   resolveLocalLabProfile,
   resolveLocalLabRole,
@@ -46083,6 +46150,7 @@ export {
   runProceduralAblationCli,
   runSealedJudge,
   runSequentialPhases,
+  runWithinCodexCreditBudget,
   safeHexEqual,
   sanitizeLoComoResultReference,
   saveBaseline,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remnic/bench",
-  "version": "9.7.15",
+  "version": "9.8.0",
   "description": "Retrieval latency ladder benchmarks + CI regression gates for @remnic/core",
   "type": "module",
   "main": "./dist/index.js",
@@ -8,6 +8,7 @@
   "exports": {
     ".": {
       "types": "./dist/index.d.ts",
+      "remnic-source": "./src/index.ts",
       "import": "./dist/index.js"
     },
     "./baselines/*": "./baselines/*",
@@ -39,8 +40,8 @@
     "hyparquet": "^1.25.7",
     "yaml": "^2.4.2",
     "zod": "^3.24.0",
-    "@remnic/coding-graph": "^9.7.15",
-    "@remnic/core": "^9.7.15"
+    "@remnic/coding-graph": "^9.8.0",
+    "@remnic/core": "^9.8.0"
   },
   "devDependencies": {
     "tsup": "^8.5.1",