@remnic/bench 9.69.33 → 9.69.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +129 -1
- package/dist/index.js +288 -70
- package/package.json +3 -3
package/dist/index.d.ts
CHANGED
|
@@ -575,6 +575,12 @@ interface BenchmarkResult {
|
|
|
575
575
|
* Must stay below the benchmark's canary floor.
|
|
576
576
|
*/
|
|
577
577
|
canaryScore?: number;
|
|
578
|
+
/**
|
|
579
|
+
* Canary floor in force when this result was produced. Artifacts written
|
|
580
|
+
* under a custom floor persist it here so tooling badges against the
|
|
581
|
+
* gate that actually ran; absent means `CANARY_SCORE_FLOOR` applied.
|
|
582
|
+
*/
|
|
583
|
+
canaryFloor?: number;
|
|
578
584
|
/** "partial" if the benchmark was interrupted; absent or "complete" otherwise. */
|
|
579
585
|
status?: "complete" | "partial";
|
|
580
586
|
/** If partial, the error that caused interruption. */
|
|
@@ -3971,6 +3977,128 @@ declare function renderBenchmarkResultExport(result: BenchmarkResult, format: Be
|
|
|
3971
3977
|
reportCardProvenance?: ReportCardProvenanceContext;
|
|
3972
3978
|
}): string;
|
|
3973
3979
|
|
|
3980
|
+
/**
|
|
3981
|
+
* Rich result summaries for tooling surfaces (bench-ui, /api/results feeds).
|
|
3982
|
+
*
|
|
3983
|
+
* `listBenchmarkResults` in results-store.ts answers "which runs exist";
|
|
3984
|
+
* this module answers "what is in a run" for display: aggregate metrics
|
|
3985
|
+
* joined with confidence intervals and effect sizes, per-task score
|
|
3986
|
+
* tables, assistant per-seed details, and the integrity badge block.
|
|
3987
|
+
*
|
|
3988
|
+
* Every summary is backed by a `BenchmarkResult` that passed
|
|
3989
|
+
* `loadBenchmarkResult` validation. Fields the validator does not check
|
|
3990
|
+
* (aggregate/statistic values, `meta.canaryFloor`, assistant
|
|
3991
|
+
* `task.details`) are still defensively coerced, so a hand-edited
|
|
3992
|
+
* artifact degrades to nulls instead of crashing a consumer.
|
|
3993
|
+
*/
|
|
3994
|
+
|
|
3995
|
+
interface BenchMetricHighlight {
|
|
3996
|
+
name: string;
|
|
3997
|
+
mean: number;
|
|
3998
|
+
}
|
|
3999
|
+
interface BenchAggregateMetric {
|
|
4000
|
+
name: string;
|
|
4001
|
+
mean: number | null;
|
|
4002
|
+
median: number | null;
|
|
4003
|
+
stdDev: number | null;
|
|
4004
|
+
min: number | null;
|
|
4005
|
+
max: number | null;
|
|
4006
|
+
ciLower: number | null;
|
|
4007
|
+
ciUpper: number | null;
|
|
4008
|
+
ciLevel: number | null;
|
|
4009
|
+
effectSize: number | null;
|
|
4010
|
+
effectInterpretation: string | null;
|
|
4011
|
+
}
|
|
4012
|
+
interface BenchTaskScoreEntry {
|
|
4013
|
+
name: string;
|
|
4014
|
+
value: number;
|
|
4015
|
+
}
|
|
4016
|
+
interface BenchPerSeedScore {
|
|
4017
|
+
seed: number;
|
|
4018
|
+
identityAccuracy: number | null;
|
|
4019
|
+
stanceCoherence: number | null;
|
|
4020
|
+
novelty: number | null;
|
|
4021
|
+
calibration: number | null;
|
|
4022
|
+
parseOk: boolean;
|
|
4023
|
+
notes: string;
|
|
4024
|
+
latencyMs: number | null;
|
|
4025
|
+
}
|
|
4026
|
+
interface BenchAssistantTaskDetails {
|
|
4027
|
+
focus: string | null;
|
|
4028
|
+
rubricId: string | null;
|
|
4029
|
+
rubricSha256: string | null;
|
|
4030
|
+
perSeedScores: BenchPerSeedScore[];
|
|
4031
|
+
judgeParseFailures: number | null;
|
|
4032
|
+
}
|
|
4033
|
+
interface BenchTaskSummary {
|
|
4034
|
+
taskId: string;
|
|
4035
|
+
question: string;
|
|
4036
|
+
expected: string;
|
|
4037
|
+
actual: string;
|
|
4038
|
+
latencyMs: number | null;
|
|
4039
|
+
totalTokens: number;
|
|
4040
|
+
primaryScore: number | null;
|
|
4041
|
+
scoreEntries: BenchTaskScoreEntry[];
|
|
4042
|
+
assistantDetails?: BenchAssistantTaskDetails | null;
|
|
4043
|
+
}
|
|
4044
|
+
type BenchIntegritySplit = "public" | "holdout" | "unknown";
|
|
4045
|
+
interface BenchIntegritySummary {
|
|
4046
|
+
/** Which split produced this result. `unknown` on legacy results. */
|
|
4047
|
+
split: BenchIntegritySplit;
|
|
4048
|
+
/** True when qrels/judge/dataset hashes are all present and well-formed. */
|
|
4049
|
+
sealsPresent: boolean;
|
|
4050
|
+
/** True when the canary score is non-null and sits at or below the floor. */
|
|
4051
|
+
canaryUnderFloor: boolean | null;
|
|
4052
|
+
/** The canary score recorded with the result, when present. */
|
|
4053
|
+
canaryScore: number | null;
|
|
4054
|
+
/** The canary floor applied — defaults to `CANARY_SCORE_FLOOR`. */
|
|
4055
|
+
canaryFloor: number;
|
|
4056
|
+
/** Truncated hashes for display (first 12 chars). */
|
|
4057
|
+
qrelsSealedHashShort: string | null;
|
|
4058
|
+
judgePromptHashShort: string | null;
|
|
4059
|
+
datasetHashShort: string | null;
|
|
4060
|
+
}
|
|
4061
|
+
interface BenchResultSummary {
|
|
4062
|
+
id: string;
|
|
4063
|
+
benchmark: string;
|
|
4064
|
+
benchmarkTier: string;
|
|
4065
|
+
timestamp: string;
|
|
4066
|
+
mode: string;
|
|
4067
|
+
totalLatencyMs: number | null;
|
|
4068
|
+
meanQueryLatencyMs: number | null;
|
|
4069
|
+
taskCount: number;
|
|
4070
|
+
metricHighlights: BenchMetricHighlight[];
|
|
4071
|
+
primaryMetric: string | null;
|
|
4072
|
+
primaryScore: number | null;
|
|
4073
|
+
runCount: number;
|
|
4074
|
+
estimatedCostUsd: number | null;
|
|
4075
|
+
totalTokens: number | null;
|
|
4076
|
+
inputTokens: number | null;
|
|
4077
|
+
outputTokens: number | null;
|
|
4078
|
+
systemProvider: string;
|
|
4079
|
+
judgeProvider: string;
|
|
4080
|
+
providerKey: string;
|
|
4081
|
+
adapterMode: string;
|
|
4082
|
+
aggregateMetrics: BenchAggregateMetric[];
|
|
4083
|
+
taskSummaries: BenchTaskSummary[];
|
|
4084
|
+
integrity: BenchIntegritySummary;
|
|
4085
|
+
assistantRubricId?: string | null;
|
|
4086
|
+
assistantRubricSha256?: string | null;
|
|
4087
|
+
assistantRunId?: string | null;
|
|
4088
|
+
filePath: string;
|
|
4089
|
+
}
|
|
4090
|
+
interface BenchResultFileWarning {
|
|
4091
|
+
filePath: string;
|
|
4092
|
+
reason: string;
|
|
4093
|
+
}
|
|
4094
|
+
interface BenchResultSummaryPayload {
|
|
4095
|
+
resultsDir: string;
|
|
4096
|
+
summaries: BenchResultSummary[];
|
|
4097
|
+
skippedFiles?: BenchResultFileWarning[];
|
|
4098
|
+
}
|
|
4099
|
+
declare function summarizeBenchmarkResult(result: BenchmarkResult, filePath: string): BenchResultSummary;
|
|
4100
|
+
declare function loadBenchmarkResultSummaries(resultsDir: string): Promise<BenchResultSummaryPayload>;
|
|
4101
|
+
|
|
3974
4102
|
interface HaystackTurn {
|
|
3975
4103
|
role: "user" | "assistant";
|
|
3976
4104
|
content: string;
|
|
@@ -9710,4 +9838,4 @@ declare function pickOne<T>(rng: SeededRandom, items: readonly T[]): T;
|
|
|
9710
9838
|
/** Deterministic Fisher-Yates shuffle returning a new array. */
|
|
9711
9839
|
declare function shuffled<T>(rng: SeededRandom, items: readonly T[]): T[];
|
|
9712
9840
|
|
|
9713
|
-
export { AMA_BENCH_DIAGNOSTIC_VARIANTS, ASSISTANT_AGENT_CONFIG_KEY, ASSISTANT_JUDGE_CONFIG_KEY, ASSISTANT_MEETING_PREP_SCENARIOS, ASSISTANT_MEETING_PREP_SMOKE_SCENARIOS, ASSISTANT_MORNING_BRIEF_SCENARIOS, ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SMOKE_SCENARIOS, ASSISTANT_RUBRIC_DIMENSIONS, ASSISTANT_RUBRIC_ID_KEY, ASSISTANT_SEEDS_CONFIG_KEY, ASSISTANT_SPOT_CHECK_DIR_KEY, ASSISTANT_SYNTHESIS_SCENARIOS, ASSISTANT_SYNTHESIS_SMOKE_SCENARIOS, type AblationConfigOverrides, type AbstentionRetrievalCase, type ActionIntentV1, ActionIntentV1Schema, type AggregateMetrics, type AmaBenchDiagnosticAdapterOptions, type AmaBenchDiagnosticAnswererMode, type AmaBenchDiagnosticBreakdown, type AmaBenchDiagnosticMatrixArtifact, type AmaBenchDiagnosticRecallMode, type AmaBenchDiagnosticRunContext, type AmaBenchDiagnosticTaskEvidence, type AmaBenchDiagnosticTaskRow, type AmaBenchDiagnosticVariant, type AmaBenchDiagnosticVariantSummary, type AnalyzeRepeatedFailureOptions, type AnthropicProviderConfig, type AssistantAgent, type AssistantMemoryFact, type AssistantMemoryGraph, type AssistantRubricDimension, type AssistantRubricRequest, type AssistantRubricScores, type AssistantRunnerOptions, type AssistantScenario, type AssistantStance, type AttackRecallOptions, type AttackRetrievalHit, type AttackerMode, type AttributeOptions, type AttributionClass, type AttributionEnvironment, type AttributionLabel, type AttributionMemory, type AttributionReport, BENCHMARK_ARTIFACT_SCHEMA_VERSION, BENCHMARK_INTEGRITY_META_SCHEMA, BENCHMARK_REPRO_MANIFEST_FILENAME, BENCHMARK_REPRO_MANIFEST_SCHEMA_VERSION, BENCHMARK_RESULT_SCHEMA, BENCHMARK_SPLIT_TYPES, BUILD_WEEK_EVIDENCE_RECEIPT_SCHEMA_VERSION, BUILD_WEEK_LIMITATIONS, type BaseTask, BaseTaskSchema, type BaselineRow, type BaselineScenario, type BeamDatasetPreview, type BenchConfig, type BenchJudge, type BenchJudgeResult, type BenchMemoryAdapter, type BenchModelSource, type BenchReasoningEffort, type BenchRecallLineageStatus, type BenchRecallOptions, type BenchRecallSupportAssessment, type BenchRecallSupportRequest, type BenchRecallSupportStatus, type BenchRecallTrace, type BenchRecallTraceCoreCapture, type BenchRecallTraceLcmCandidate, type BenchRecallTraceRange, type BenchRecallTraceSection, type BenchRecallTraceSelection, type BenchRecallWithTraceResult, type BenchResponder, type BenchResponse, type BenchRuntimeProfile, type BenchTier, type BenchmarkArtifact, type BenchmarkArtifactEnvironment, type BenchmarkArtifactHardware, type BenchmarkArtifactJudgeCalibration, type BenchmarkArtifactPerTaskScore, type BenchmarkArtifactSystem, type BenchmarkArtifactTier, type BenchmarkCategory, type BenchmarkDefinition, type BenchmarkExecutionProvenance, type BenchmarkIntegrityMeta, type BenchmarkMeta, type BenchmarkMode, type BenchmarkReport, type BenchmarkReproManifest, type BenchmarkReproManifestDataset, type BenchmarkReproManifestFile, type BenchmarkReproManifestResult, type BenchmarkReproManifestSupplementalArtifact, type BenchmarkResult, type BenchmarkSplitType, type BenchmarkStatus, type BenchmarkSuiteResult, type BenchmarkTier, type BootstrapKappaOptions, type BootstrapKappaResult, type BuildBenchmarkArtifactInput, type BuildBenchmarkPublishFeedOptions, type BuildBenchmarkReproManifestOptions, type BuildBuildWeekEvidenceReceiptOptions, type BuildWeekEvidenceReceipt, type BuildWeekEvidenceReceiptProvider, type BuildWeekLimitationCode, type BuiltInProvider, CALIBRATION_SLICE_SIZE, CANARY_FIXED_RECALL, CANARY_SCORE_FLOOR, DEFAULT_10K_FIXTURE as CODING_GRAPH_10K_FIXTURE, CODING_GRAPH_BENCH_SCHEMA_VERSION, DEFAULT_TOLERANCE_PERCENT as CODING_GRAPH_DEFAULT_TOLERANCE, MIN_ITERATIONS as CODING_GRAPH_MIN_ITERATIONS, DEFAULT_SMOKE_FIXTURE as CODING_GRAPH_SMOKE_FIXTURE, type CalibrationAnswer, type CalibrationVerdictPair, type CanaryAdapterOptions, type CanaryFloorCheck, type CaptureLoCoMoRetrievalTraceOptions, type ClaudeCliProviderConfig, type CodexCliNativeUsage, type CodexCliProviderConfig, CodexCreditAccountingError, type CodexCreditBudgetConfig, CodexCreditDispatchError, type CodexCreditReceipt, type CodexCreditReceiptScope, type CodexCreditReconciliationReceipt, type CodingGraphBaseline, type CodingGraphBenchConfig, type CodingGraphBenchReport, type MachineFingerprint as CodingGraphMachineFingerprint, type CodingGraphMetricKey, type RegressionMetricDetail as CodingGraphRegressionDetail, type RegressionMetricKey as CodingGraphRegressionKey, type RegressionGateResult as CodingGraphRegressionResult, type CohenKappaResult, type ComparisonMetricDelta, type ComparisonResult, type CompletionOpts, type CompletionResult, type ConfidenceInterval, type ContaminationCheckResult, type ContaminationEntry, type ContaminationManifest, type ControlledGateDecision, type ControlledResponsesAgentDriverConfig, type ControlledResponsesCaps, type ControlledResponsesDisposition, ControlledResponsesDriver, type ControlledResponsesDriverConfig, type ControlledResponsesEpisodeInput, type ControlledResponsesEpisodeResult, type ControlledResponsesFault, type ControlledResponsesResponseEvent, type ControlledResponsesToolDefinition, type ControlledResponsesToolEvent, type ControlledResponsesTransport, type CustomBenchmarkScoring, type CustomBenchmarkSpec, type CustomBenchmarkTask, DATASET_SPLITS, DEFAULT_ABLATION_BENCHMARK, DEFAULT_ABLATION_BOOTSTRAP_SEED, DEFAULT_ASSISTANT_RUBRIC_ID, DEFAULT_BASELINE_SCENARIOS, DEFAULT_JUDGE_BINARIZATION_THRESHOLD, DEFAULT_KAPPA_BOOTSTRAP_SAMPLES, DEFAULT_KAPPA_CONFIDENCE_LEVEL, DEFAULT_OPENAI_RESPONSES_JUDGE_MODEL, DRIFT_GEN_DEFAULTS, DRIFT_GEN_VERSION, type DatasetSource, type DatasetSplit, type DiagnoseLoComoProfileDeltaOptions, type DiagnoseLoComoRecallDeltaOptions, type DiscoveredModel, type DriftGenAuditRecord, type DriftGenCorpus, type DriftGenManifest, type DriftGenOptions, type DriftGenResult, type DriftSession, type DriftSessionTurn, type DriftValidationReport, type DriftValidationStats, EMPTY_CONTAMINATION_MANIFEST, type EffectSizeInterpretation, type EffectSizeSummary, type EvaluateTaskStateOptions, type ExplainResult, type ExtractedEntity, type ExtractedLink, type ExtractedPage, type ExtractionAttackOptions, type ExtractionAttackResult, type ExtractionAttackTarget, type FixtureGenerator, type FixtureOutput, type FixtureVariant, GATE_STATUSES, GENERAL_ANSWER_JUDGE_RUBRIC, type GateStatus, type GeneratedFile, type GeneratedRepo, type GoldEntity, type GoldEntityType, type GoldFact, type GoldFactKind, type GoldGraph, type GoldLink, type GoldMemoryAttribution, type GoldPage, type GoldProbe, type GoldProbeCategory, type H6BenchmarkDataset, H6BenchmarkDatasetSchema, type H6TrapId, H6_ACTION_INTENT_JSON_SCHEMA, H6_ARMS, H6_DATASET_JSON_SCHEMA, H6_DECISION_RULE, H6_FROZEN_INVENTORY_HASH, H6_FROZEN_SEED, H6_FROZEN_SPLITS, H6_SUPPORT_ARTIFACT_PATHS, H6_TASK_JSON_SCHEMA, H6_TRAP_FINGERPRINT_JSON_SCHEMA, H6_TRAP_IDS, HOST_FAULT_RETRY_LIMIT, type HarnessRng, INJECTION_SUITE_ARMS, INJECTION_SUITE_FAMILIES, INJECTION_SUITE_VERSION, INTEGRITY_CIPHER_ALGORITHM, INTEGRITY_HASH_ALGORITHM, INTEGRITY_META_FIELDS, INVENTED_DOMAINS, type IngestionBenchAdapter, type IngestionLog, type InjectionSuiteArm, type InjectionSuiteCliInput, type InjectionSuiteCliResult, type InjectionSuiteEpisodeRow, type InjectionSuiteFamily, type InjectionSuiteRowIdentity, JUDGE_CALIBRATION_KAPPA_THRESHOLD, JUDGE_CALIBRATION_PROTOCOL_VERSION, type JudgeCalibrationCheckpointProvenance, type JudgeCalibrationIdentities, type JudgeCalibrationResult, type JudgeCategory, type KappaConfidenceInterval, LOCAL_LAB_PROVIDER_KINDS, LOCOMO_DATASET_FILENAMES, LOCOMO_FULL_TASK_COUNT, LOCOMO_RECALL_DIFF_LINE_LIMIT, LOCOMO_RECALL_EXCERPT_CHARS, LOCOMO_RETRIEVAL_TRACE_BUDGET_VERSION, LOCOMO_RETRIEVAL_TRACE_DELTA_SCHEMA_VERSION, LOCOMO_RETRIEVAL_TRACE_SCHEMA_VERSION, LOCOMO_RETRIEVAL_TRACE_SELECTION_VERSION, LONG_MEM_EVAL_DATASET_FILENAMES, type LeaderboardArtifactWrite, type LettaAdapterConfig, LettaMemCorrectAdapter, type LlmJudge, type LlmProvider, type LoCoMoCategory, type LoCoMoRetrievalMechanism, type LoCoMoRetrievalMechanismSummary, type LoCoMoRetrievalSessionReceipt, type LoCoMoRetrievalStructuralTrace, type LoCoMoRetrievalTaskDelta, type LoCoMoRetrievalTaskReceipt, type LoCoMoRetrievalTraceCoreCaptureReceipt, type LoCoMoRetrievalTraceDeltaReport, type LoCoMoRetrievalTraceProfile, type LoCoMoRetrievalTraceReceipt, type LoCoMoRetrievalTraceSelectionManifest, type LoCoMoRetrievalTraceSelector, type LoCoMoStructuralMultisetDelta, type LoComoCategoryDelta, type LoComoFinalContextRegression, type LoComoMetricDelta, type LoComoProfileArtifactEvidence, type LoComoProfileDeltaReport, type LoComoRawResultEvidence, type LoComoRecallCategoryDelta, type LoComoRecallContextSummary, type LoComoRecallDeltaReport, type LoComoRecallLineDelta, type LoComoRecallLineEvidence, type LoComoRecallMetricDelta, type LoComoRecallResultProvenance, type LoComoRecallTextDigest, type LoComoTaskRegression, type LoadDatasetOptions, type LoadSealedQrelsOptions, type LoadedDataset, type LoadedJudgeCalibrationState, type LocalLabManifest, type LocalLabManifestNotes, type LocalLabPhase, type LocalLabPhaseDescriptor, type LocalLabPhaseExecute, type LocalLabPhaseName, type LocalLabPhaseOutcome, LocalLabPreflightError, type LocalLabPreflightFailure, type LocalLabPreflightInput, type LocalLabPreflightOptions, type LocalLabPreflightResult, type LocalLabPreflightSuccess, type LocalLabProviderKind, type LocalLabRoleConfig, type LocalLlmProviderConfig, MAX_ROW_ATTEMPTS, MEMCORRECT_CORRECTION_ACCEPTANCE_RUBRIC, MEMCORRECT_CORRECTION_ACCEPTANCE_RUBRIC_VERSION, MEMCORRECT_STALE_HARM_RUBRIC, MEMCORRECT_STALE_HARM_RUBRIC_VERSION, MEMORY_EVAL_DIMENSIONS, MEMORY_EVAL_PUBLIC_LINE, MIN_CALIBRATION_SOURCE_TASKS, MITIGATED_BASELINE_SCENARIOS, type MaterializeOptions, type MaterializedRepo, type McpArgumentSemantic, type McpBackendErrorCode, type McpBackendResult, type McpBenchMemoryAdapter, type McpConformanceResult, type McpHttpTransportConfig, type McpListedTool, type McpMemCorrectAdapter, type McpMemoryAdapterOptions, McpMemoryBackendError, type McpMemoryToolMapping, type McpMemoryTransportConfig, type McpStdioTransportConfig, type McpToolCallResult, type McpToolClient, type McpToolMappingEntry, type McpToolMappingValue, type McpToolOperation, type Mem0AdapterConfig, Mem0MemCorrectAdapter, type MemCorrectGeneratorOptions, type MemCorrectJudgeRequest, type MemCorrectJudgeResult, type MemCorrectSystemAdapter, type MemoryEvalCategory, type MemoryEvalDimension, type MemoryEvalDimensionId, type MemoryEvalMetric, type MemoryGraph, type MemoryStats, type MemorySystem, type Message, type MetricAggregate, type MicroMetric, MissingCredentialError, type MitigatedBaselineConfig, type MitigatedTargetConfig, type MultipleChoiceQuestion, OPENAI_RESPONSES_JUDGE_RUBRIC_VERSION, OTHER_NAMESPACE_MEMORIES, type OllamaChatMessage, type OllamaProviderConfig, type OpenAiCompatibleProviderConfig, OpenAiResponsesJudgeError, type OpenAiResponsesJudgeErrorCode, type OpenAiResponsesJudgeTelemetry, OpenAiResponsesProvider, type OpenAiResponsesProviderConfig, type OpenAiResponsesVerdict, type OpenAiResponsesVerdictResult, PROCEDURAL_REAL_SCENARIOS, PROCEDURAL_REAL_SCENARIOS_SMOKE, PUBLISHED_BENCHMARK_ARTIFACT_IDS, type PairedAnswerReplayCache, type PairedAnswerReplayEntry, type ParsedOllamaChatResponse, type PersonalizationRetrievalCase, type PreflightDiscoveredModel, type ProceduralAblationArtifact, type ProceduralAblationPerCase, type ProceduralAblationScenario, type ProceduralRealScenario, type ProceduralRealScenarioCategory, type ProviderBaseConfig, type ProviderConfig, type ProviderDiscoveryResult, type ProviderFactoryConfig, type PublishSkipReason, type PublishSkipRecord, type PublishedBenchmarkFeed, type PublishedBenchmarkFeedEntry, type PublishedBenchmarkId, REPEATED_FAILURE_ARMS, REPEATED_FAILURE_CONFIDENCE_LEVEL, REPEATED_FAILURE_INVALID_REASONS, REPEATED_FAILURE_STATISTICS_DRAWS, REQUIRED_FRONTMATTER_FIELDS, type RecallMetrics, type RecoveredMemory, type RegressionDetail, type RegressionGateResult$1 as RegressionGateResult, type RemnicAdapterOptions, type RepeatedFailureActionEvaluator, type RepeatedFailureArm, type RepeatedFailureCheckpointLoadResult, type RepeatedFailureCliCommandResult, type RepeatedFailureEffectAnalysis, type RepeatedFailureEpisode, type RepeatedFailureEpisodeDriver, type RepeatedFailureEpisodeEvidence, type RepeatedFailureEpisodeInput, type RepeatedFailureEpisodeRow, type RepeatedFailureExpectedDesign, type RepeatedFailureFactPairAudit, type RepeatedFailureFinalRepoEvidence, type RepeatedFailureFinalState, type RepeatedFailureGateEvent, type RepeatedFailureHolmResult, type RepeatedFailureInterval, type RepeatedFailureInvalidReason, type RepeatedFailureIsolationIdentity, type RepeatedFailureLocalToolHost, type RepeatedFailureNullableInterval, RepeatedFailureOllamaChatDriver, type RepeatedFailureOllamaChatDriverConfig, type RepeatedFailureProposedAction, type RepeatedFailureRowCheckpoint, type RepeatedFailureRowClaim, type RepeatedFailureRowIdentity, RepeatedFailureRowStore, type RepeatedFailureRowStoreOptions, type RepeatedFailureRunMetadata, type RepeatedFailureStatisticalAnalysis, type RepeatedFailureSuiteManifest, type RepeatedFailureSupportDecision, type RepeatedFailureTaskCut, type RepeatedFailureTimidityAnalysis, type RepeatedFailureTokenUsage, type RepeatedFailureTokenizer, type RepeatedFailureToolDefinition, type RepeatedFailureToolExecutionResult, type RepeatedFailureTrapAuditArtifact, type RepeatedFailureTrapAuditExpected, type RepeatedFailureTrapAuditMetrics, type RepeatedFailureTrapAuditRow, type RepeatedFailureTrapAuditRowIdentity, type RepeatedFailureTrapAuditThresholds, type RepeatedFailureTry, type ReplayRepeatedFailureStatisticsOptions, type ReportCardProvenanceContext, type ResolveBenchRuntimeProfileOptions, type ResolvedBenchRuntimeProfile, type ResolvedLocalLabProfile, type ResolvedLocalLabRole, type ResolvedRunBenchmarkOptions, type ResponsesApiOutputItem, type ResponsesApiRequest, type ResponsesApiResponse, type ResponsesApiUsage, type RetrievalMissStage, type RevisionShas, type RotatedChoices, type RunBenchmarkOptions, type RunJudgeCalibrationOptions, type RunProceduralAblationCliArgs, type RunProceduralAblationOptions, type RunRepeatedFailureCliCommandInput, type RunRepeatedFailureSuiteOptions, type RunRepeatedFailureSuiteResult, type RunSequentialPhasesOptions, type RunTrapAuditOptions, SCHEMA_TIER_FIXTURE, SCHEMA_TIER_SMOKE_FIXTURE, SEALED_PROMPT_REGISTRY, SINGLE_FLAG_ABLATION_MATRIX, STATE_CLASSIFICATIONS, SYNTHETIC_MEMORIES, type SanitizedDiagnosticProvider, type SavedBaseline, type SchemaTierCorpus, type SchemaTierFixture, type SchemaTierName, type SchemaTierPage, type SchemaTierPageFrontmatter, type SealedArtifact, type SealedJudgeDecision, type SealedJudgeInput, type SealedQrelsArtifact, type SealedQrelsHandle, type SealedRubric, type SearchResult, type SeededMemory, type SeededRandom, type SeededRng, type SequentialPhaseHooks, type SingleFlagAblationCell, type SingleFlagAblationId, type SpotCheckLogger, type StageObservation, type StageStatus, type StateClassification, type StateEvaluationResult, type StatisticalReport, type StrategyPatch, StrategyPatchSchema, type StructuredJudge, StructuredJudgeError, type StructuredJudgeErrorCode, type StructuredJudgeProvider, type StructuredJudgeTelemetry, type StructuredJudgeVerdict, type StructuredJudgeVerdictResult, type StructuredVerdictRequest, type SyntheticEdge, type SyntheticEmailIngestionAdapterOptions, type SyntheticFile, type SyntheticFileIR, SyntheticFileSchema, type SyntheticRepoConfig, type SyntheticSymbol, type SyntheticTargetOptions, TRAP_TAXONOMY, type TaskAttribution, type TaskAttributionGoldWitnessV1, type TaskAttributionRetrievalWitnessV1, type TaskAttributionWitness, type TaskAttributionWitnessRuntimeV1, type TaskAttributionWitnessV1, type TaskResult, type TaskTokenUsage, type TaskVariant, TaskVariantSchema, type TemporalRetrievalCase, type ThirdPartyAdapterConfig, type TierDetail, type TimelineEntry, type TokenUsage, type TrapFingerprintV1, TrapFingerprintV1Schema, type TrapTaxonomyItem, TrapTaxonomyItemSchema, type ValidationIssue, type ValidationReport, type WallMetric, type WriteBenchmarkArtifactResult, type ZepAdapterConfig, ZepMemCorrectAdapter, addContaminationEntry, aggregateTaskScores, analyzeRepeatedFailureRows, answerBenchmarkQuestion, applyPatchAndCommit, assertCanaryUnderFloor, assertIntegrityMetaPresent, assertPublishableIntegrity, assertSha256Hex, assertTrapDatasetPreflight, assistantMeetingPrepDefinition, assistantMorningBriefDefinition, assistantNextBestActionDefinition, assistantSynthesisDefinition, attributeGoldMemory, attributeRun, attributeTask, backlinkF1, binarizeJudgeScore, bootstrapCohensKappaConfidenceInterval, bootstrapMeanConfidenceInterval, buildAmaBenchDiagnosticMatrixArtifact, buildAmaBenchDiagnosticVariantSummary, buildAmaBenchLeaderboardRows, buildBaselineFromReport, buildBenchmarkArtifact, buildBenchmarkArtifactFilename, buildBenchmarkPublishFeed, buildBenchmarkReproManifest, buildBenchmarkRunSeeds, buildBuildWeekEvidenceReceipt, buildCodexCreditReceipt, buildDriftCorpus, buildJudgePayload, buildOracleTrajectoryRecall, buildProviderFreeLoCoMoRetrievalConfig, buildRepeatedFailureRowKey, buildSchemaTierFixture, buildSchemaTierSmokeFixture, calculateCodexBudgetUnits, calculateJaccardSimilarity, calendarFixture, canonicalJsonStringify, captureBenchmarkExecutionProvenance, captureLoCoMoRetrievalTrace, captureMachineFingerprint, chatFixture, checkCodingGraphRegression, checkDatasetContamination, checkRegression, clampScore, cohensD, compareResults, computeBenchmarkReproDatasetInventoryHash, computeBenchmarkReproManifestArtifactHash, computeCohensKappa, computeH6InventoryHash, computeH6SupportArtifactHashes, computeRevisionShas, computeSealHash, computeTrapAuditArtifactHash, computeTrapAuditMetrics, containsAnswer, createSeededRng$1 as createAdamSeededRng, createAmaBenchDiagnosticAdapter, createAnthropicProvider, createCanaryAdapter, createClaudeCliProvider, createCodexCliProvider, createSeededRng as createCodingGraphSeededRng, createControlledResponsesAgentDriver, createDeterministicSpotCheckLogger, createGatewayResponder, createLightweightAdapter, createLiteLlmProvider, createLocalLlmProvider, createMcpDemoMemCorrectAdapter, createMcpDemoMemoryAdapter, createMcpMemCorrectAdapter, createMcpMemoryAdapter, createMitigatedTarget, createOllamaProvider, createOpenAiCompatibleProvider, createOpenAiResponsesBenchJudge, createOpenAiResponsesProvider, createSeededRandom$1 as createProceduralAblationSeededRandom, createProvider, createProviderBackedAmaBenchRecommendedJudge, createProviderBackedJudge, createProviderBackedResponder, createProviderBackedStructuredJudge, createRemnicAdapter, createRepeatedFailureOllamaChatDriver, createResponderFromProvider, createSeededRandom, createSeededRng$2 as createSeededRng, createSpotCheckFileLogger, createStructuredBenchJudge, createStructuredJudgeFromProvider, createSyntheticEmailIngestionAdapter, createSyntheticTarget, createTimeoutGuardedAdapter, decideRepeatedFailureContent, decideRepeatedFailureStudy, decideRepeatedFailureTiming, defaultBenchmarkBaselineDir, defaultBenchmarkPublishPath, deleteBenchmarkResults, diagnoseLoCoMoRetrievalTraceDelta, diagnoseLoComoProfileDelta, diagnoseLoComoRecallDelta, discoverAllProviders, discoveryEndpointFor, emailFixture, entityRecall, evaluateTaskState, exactMatch, executeLocalRow, extractMetrics as extractCodingGraphMetrics, extractContentWords, extractMarkdownSectionsByTitle, f1Score, fixtureToAblationScenarios, formatHandoffNote, formatMissingDatasetError, generateDriftCorpus, generateFamilyVariants, generateH6BenchmarkDataset, generateReport, generateSuiteVariants, generateSyntheticRepo, getAblationCell, getBenchmark, getBenchmarkLowerIsBetter, getGitSha, getMemoryEvalDimension, getProviderBackedJudgePromptIdentity, getRemnicVersion, getTrapTaxonomyItem, hashBenchmarkArtifact, hashBytes, hashCanonicalJson, hashOrderedQuestionIds, hashString, holmAdjust, injectionSuiteResumeContractHash, integrityMetaIsComplete, interpretEffectSize, isAmaBenchUnknownLikeAnswer, isContaminationEntry, isContaminationManifest, isRepeatedFailureTimidityEquivalent, isSafeSyntheticPath, isSealedQrelsArtifact, isSha256Hex, isStructuredJudgeProvider, isTaskFailed, judgeMemCorrectCorrectionAcceptance, judgeMemCorrectStaleMemoryHarm, lexicalSimilarity, linkMatches, listBenchmarkBaselines, listBenchmarkResults, listBenchmarks, listMemoryEvalBenchmarkIds, listMemoryEvalDimensions, llmJudgeScore, llmJudgeScoreDetailed, loadAblationFixture, loadBaseline, loadBeamDatasetPreview, loadBenchmarkArtifact, loadBenchmarkBaseline, loadBenchmarkReportCardProvenance, loadBenchmarkResult, loadCommittedH6BenchmarkDataset, loadCustomBenchmarkFile, loadJudgeCalibrationState, loadLoCoMo10, loadLocalLabManifest, loadLongMemEvalS, loadSealKeyFromEnv, loadSealedQrels, loadSealedRubric, matchEntity, materializeTaskRepo, mergeContaminationManifests, openSeal, orchestrateBenchmarkRuns, pairedDeltaConfidenceInterval, parseBenchmarkArtifact, parseCodexJsonlUsage, parseCustomBenchmark, parseLocalLabManifest, parseRepeatedFailureEpisodeRow, parseRubricResponse, parseSealedQrels, pickOne, pickStableQualifiedName, planInjectionSuiteRows, precisionAtK, preflightLoCoMoRetrievalTraceCapture, preflightLocalLabRole, projectFolderFixture, randomInt, recallAtK, reconcileCodexCreditLedger, redactBenchmarkResultSecrets, relativeRiskReduction, renderAttributionReportTable, renderBaselineMarkdown, renderBenchmarkResultExport, renderLoComoProfileDeltaMarkdown, renderLoComoRecallDeltaMarkdown, renderMemorySummaryForJudge, renderMemoryViewForAgent, replayRepeatedFailureStatistics, resolveAssistantAgent, resolveAssistantRubricId, resolveAssistantSeeds, resolveAssistantSpotCheckDir, resolveBenchRuntimeProfile, resolveBenchmarkPhaseTimeoutMs, resolveBenchmarkProgressLogging, resolveBenchmarkResultReference, resolveBenchmarkRunCount, resolveBenchmarkRunId, resolveCodexCreditBudgetConfig, resolveCommittedH6FixtureDirectory, resolveLocalLabJudgeProviderConfig, resolveLocalLabProfile, resolveLocalLabRole, resolveStructuredJudge, rotateDistractors, rougeL, runAssistantBenchmark, runAssistantMeetingPrepBenchmark, runAssistantMorningBriefBenchmark, runAssistantNextBestActionBenchmark, runAssistantSynthesisBenchmark, runAttributeCliCommand, runBaseline, runBenchSuite, runBenchmark, runCodingGraphBenchmark, runCustomBenchmarkFile, runDriftGenCliCommand, runExplain, runExtractionAttack, runInjectionSuiteCliCommand, runJudgeCalibration, runMitigatedBaseline, runProceduralAblation, runProceduralAblationCli, runRepeatedFailureCliCommand, runRepeatedFailurePaperReportCliCommand, runRepeatedFailureSuite, runSealedJudge, runSequentialPhases, runTrapAudit, runTrapAuditCliCommand, runWithinCodexCreditBudget, safeHexEqual, sanitizeBenchmarkResultForJson, sanitizeLoComoResultReference, saveBaseline, saveBenchmarkBaseline, schemaCompleteness, sealPayload, selectAmaBenchDiagnosticVariants, selectCalibrationSlice, selectFixtureVariant, serializeAttributionReport, serializeBenchmarkArtifact, serializeBuildWeekEvidenceReceipt, serializeH6FixtureJson, serializeJsonl, serializeLoCoMoRetrievalTraceDelta, serializeLoCoMoRetrievalTraceReceipt, serializeSealedQrels, shuffleTasks, shuffled, timed, tokenizeContent, unresolvedHelperImports, validateDriftCorpus, validateH6Dataset, validateH6FixtureBundle, validateH6StateDefiningIndependence, validateOllamaChatEndpoint, verifyMatchingTrapAudit, verifyRubricDigest, verifyTrapAuditArtifact, writeBenchmarkArtifact, writeBenchmarkPublishFeed, writeBenchmarkReproManifest, writeBenchmarkResult, writeBuildWeekEvidenceReceipt, writeH6FixtureBundle, writeJudgeCalibrationState, writeLeaderboardArtifactsForResult, writeRepeatedFailurePaperArtifacts, writeRepeatedFailureRunMetadata, writeRepeatedFailureStatistics, zeroScores };
|
|
9841
|
+
export { AMA_BENCH_DIAGNOSTIC_VARIANTS, ASSISTANT_AGENT_CONFIG_KEY, ASSISTANT_JUDGE_CONFIG_KEY, ASSISTANT_MEETING_PREP_SCENARIOS, ASSISTANT_MEETING_PREP_SMOKE_SCENARIOS, ASSISTANT_MORNING_BRIEF_SCENARIOS, ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SMOKE_SCENARIOS, ASSISTANT_RUBRIC_DIMENSIONS, ASSISTANT_RUBRIC_ID_KEY, ASSISTANT_SEEDS_CONFIG_KEY, ASSISTANT_SPOT_CHECK_DIR_KEY, ASSISTANT_SYNTHESIS_SCENARIOS, ASSISTANT_SYNTHESIS_SMOKE_SCENARIOS, type AblationConfigOverrides, type AbstentionRetrievalCase, type ActionIntentV1, ActionIntentV1Schema, type AggregateMetrics, type AmaBenchDiagnosticAdapterOptions, type AmaBenchDiagnosticAnswererMode, type AmaBenchDiagnosticBreakdown, type AmaBenchDiagnosticMatrixArtifact, type AmaBenchDiagnosticRecallMode, type AmaBenchDiagnosticRunContext, type AmaBenchDiagnosticTaskEvidence, type AmaBenchDiagnosticTaskRow, type AmaBenchDiagnosticVariant, type AmaBenchDiagnosticVariantSummary, type AnalyzeRepeatedFailureOptions, type AnthropicProviderConfig, type AssistantAgent, type AssistantMemoryFact, type AssistantMemoryGraph, type AssistantRubricDimension, type AssistantRubricRequest, type AssistantRubricScores, type AssistantRunnerOptions, type AssistantScenario, type AssistantStance, type AttackRecallOptions, type AttackRetrievalHit, type AttackerMode, type AttributeOptions, type AttributionClass, type AttributionEnvironment, type AttributionLabel, type AttributionMemory, type AttributionReport, BENCHMARK_ARTIFACT_SCHEMA_VERSION, BENCHMARK_INTEGRITY_META_SCHEMA, BENCHMARK_REPRO_MANIFEST_FILENAME, BENCHMARK_REPRO_MANIFEST_SCHEMA_VERSION, BENCHMARK_RESULT_SCHEMA, BENCHMARK_SPLIT_TYPES, BUILD_WEEK_EVIDENCE_RECEIPT_SCHEMA_VERSION, BUILD_WEEK_LIMITATIONS, type BaseTask, BaseTaskSchema, type BaselineRow, type BaselineScenario, type BeamDatasetPreview, type BenchAggregateMetric, type BenchAssistantTaskDetails, type BenchConfig, type BenchIntegritySplit, type BenchIntegritySummary, type BenchJudge, type BenchJudgeResult, type BenchMemoryAdapter, type BenchMetricHighlight, type BenchModelSource, type BenchPerSeedScore, type BenchReasoningEffort, type BenchRecallLineageStatus, type BenchRecallOptions, type BenchRecallSupportAssessment, type BenchRecallSupportRequest, type BenchRecallSupportStatus, type BenchRecallTrace, type BenchRecallTraceCoreCapture, type BenchRecallTraceLcmCandidate, type BenchRecallTraceRange, type BenchRecallTraceSection, type BenchRecallTraceSelection, type BenchRecallWithTraceResult, type BenchResponder, type BenchResponse, type BenchResultFileWarning, type BenchResultSummary, type BenchResultSummaryPayload, type BenchRuntimeProfile, type BenchTaskScoreEntry, type BenchTaskSummary, type BenchTier, type BenchmarkArtifact, type BenchmarkArtifactEnvironment, type BenchmarkArtifactHardware, type BenchmarkArtifactJudgeCalibration, type BenchmarkArtifactPerTaskScore, type BenchmarkArtifactSystem, type BenchmarkArtifactTier, type BenchmarkCategory, type BenchmarkDefinition, type BenchmarkExecutionProvenance, type BenchmarkIntegrityMeta, type BenchmarkMeta, type BenchmarkMode, type BenchmarkReport, type BenchmarkReproManifest, type BenchmarkReproManifestDataset, type BenchmarkReproManifestFile, type BenchmarkReproManifestResult, type BenchmarkReproManifestSupplementalArtifact, type BenchmarkResult, type BenchmarkSplitType, type BenchmarkStatus, type BenchmarkSuiteResult, type BenchmarkTier, type BootstrapKappaOptions, type BootstrapKappaResult, type BuildBenchmarkArtifactInput, type BuildBenchmarkPublishFeedOptions, type BuildBenchmarkReproManifestOptions, type BuildBuildWeekEvidenceReceiptOptions, type BuildWeekEvidenceReceipt, type BuildWeekEvidenceReceiptProvider, type BuildWeekLimitationCode, type BuiltInProvider, CALIBRATION_SLICE_SIZE, CANARY_FIXED_RECALL, CANARY_SCORE_FLOOR, DEFAULT_10K_FIXTURE as CODING_GRAPH_10K_FIXTURE, CODING_GRAPH_BENCH_SCHEMA_VERSION, DEFAULT_TOLERANCE_PERCENT as CODING_GRAPH_DEFAULT_TOLERANCE, MIN_ITERATIONS as CODING_GRAPH_MIN_ITERATIONS, DEFAULT_SMOKE_FIXTURE as CODING_GRAPH_SMOKE_FIXTURE, type CalibrationAnswer, type CalibrationVerdictPair, type CanaryAdapterOptions, type CanaryFloorCheck, type CaptureLoCoMoRetrievalTraceOptions, type ClaudeCliProviderConfig, type CodexCliNativeUsage, type CodexCliProviderConfig, CodexCreditAccountingError, type CodexCreditBudgetConfig, CodexCreditDispatchError, type CodexCreditReceipt, type CodexCreditReceiptScope, type CodexCreditReconciliationReceipt, type CodingGraphBaseline, type CodingGraphBenchConfig, type CodingGraphBenchReport, type MachineFingerprint as CodingGraphMachineFingerprint, type CodingGraphMetricKey, type RegressionMetricDetail as CodingGraphRegressionDetail, type RegressionMetricKey as CodingGraphRegressionKey, type RegressionGateResult as CodingGraphRegressionResult, type CohenKappaResult, type ComparisonMetricDelta, type ComparisonResult, type CompletionOpts, type CompletionResult, type ConfidenceInterval, type ContaminationCheckResult, type ContaminationEntry, type ContaminationManifest, type ControlledGateDecision, type ControlledResponsesAgentDriverConfig, type ControlledResponsesCaps, type ControlledResponsesDisposition, ControlledResponsesDriver, type ControlledResponsesDriverConfig, type ControlledResponsesEpisodeInput, type ControlledResponsesEpisodeResult, type ControlledResponsesFault, type ControlledResponsesResponseEvent, type ControlledResponsesToolDefinition, type ControlledResponsesToolEvent, type ControlledResponsesTransport, type CustomBenchmarkScoring, type CustomBenchmarkSpec, type CustomBenchmarkTask, DATASET_SPLITS, DEFAULT_ABLATION_BENCHMARK, DEFAULT_ABLATION_BOOTSTRAP_SEED, DEFAULT_ASSISTANT_RUBRIC_ID, DEFAULT_BASELINE_SCENARIOS, DEFAULT_JUDGE_BINARIZATION_THRESHOLD, DEFAULT_KAPPA_BOOTSTRAP_SAMPLES, DEFAULT_KAPPA_CONFIDENCE_LEVEL, DEFAULT_OPENAI_RESPONSES_JUDGE_MODEL, DRIFT_GEN_DEFAULTS, DRIFT_GEN_VERSION, type DatasetSource, type DatasetSplit, type DiagnoseLoComoProfileDeltaOptions, type DiagnoseLoComoRecallDeltaOptions, type DiscoveredModel, type DriftGenAuditRecord, type DriftGenCorpus, type DriftGenManifest, type DriftGenOptions, type DriftGenResult, type DriftSession, type DriftSessionTurn, type DriftValidationReport, type DriftValidationStats, EMPTY_CONTAMINATION_MANIFEST, type EffectSizeInterpretation, type EffectSizeSummary, type EvaluateTaskStateOptions, type ExplainResult, type ExtractedEntity, type ExtractedLink, type ExtractedPage, type ExtractionAttackOptions, type ExtractionAttackResult, type ExtractionAttackTarget, type FixtureGenerator, type FixtureOutput, type FixtureVariant, GATE_STATUSES, GENERAL_ANSWER_JUDGE_RUBRIC, type GateStatus, type GeneratedFile, type GeneratedRepo, type GoldEntity, type GoldEntityType, type GoldFact, type GoldFactKind, type GoldGraph, type GoldLink, type GoldMemoryAttribution, type GoldPage, type GoldProbe, type GoldProbeCategory, type H6BenchmarkDataset, H6BenchmarkDatasetSchema, type H6TrapId, H6_ACTION_INTENT_JSON_SCHEMA, H6_ARMS, H6_DATASET_JSON_SCHEMA, H6_DECISION_RULE, H6_FROZEN_INVENTORY_HASH, H6_FROZEN_SEED, H6_FROZEN_SPLITS, H6_SUPPORT_ARTIFACT_PATHS, H6_TASK_JSON_SCHEMA, H6_TRAP_FINGERPRINT_JSON_SCHEMA, H6_TRAP_IDS, HOST_FAULT_RETRY_LIMIT, type HarnessRng, INJECTION_SUITE_ARMS, INJECTION_SUITE_FAMILIES, INJECTION_SUITE_VERSION, INTEGRITY_CIPHER_ALGORITHM, INTEGRITY_HASH_ALGORITHM, INTEGRITY_META_FIELDS, INVENTED_DOMAINS, type IngestionBenchAdapter, type IngestionLog, type InjectionSuiteArm, type InjectionSuiteCliInput, type InjectionSuiteCliResult, type InjectionSuiteEpisodeRow, type InjectionSuiteFamily, type InjectionSuiteRowIdentity, JUDGE_CALIBRATION_KAPPA_THRESHOLD, JUDGE_CALIBRATION_PROTOCOL_VERSION, type JudgeCalibrationCheckpointProvenance, type JudgeCalibrationIdentities, type JudgeCalibrationResult, type JudgeCategory, type KappaConfidenceInterval, LOCAL_LAB_PROVIDER_KINDS, LOCOMO_DATASET_FILENAMES, LOCOMO_FULL_TASK_COUNT, LOCOMO_RECALL_DIFF_LINE_LIMIT, LOCOMO_RECALL_EXCERPT_CHARS, LOCOMO_RETRIEVAL_TRACE_BUDGET_VERSION, LOCOMO_RETRIEVAL_TRACE_DELTA_SCHEMA_VERSION, LOCOMO_RETRIEVAL_TRACE_SCHEMA_VERSION, LOCOMO_RETRIEVAL_TRACE_SELECTION_VERSION, LONG_MEM_EVAL_DATASET_FILENAMES, type LeaderboardArtifactWrite, type LettaAdapterConfig, LettaMemCorrectAdapter, type LlmJudge, type LlmProvider, type LoCoMoCategory, type LoCoMoRetrievalMechanism, type LoCoMoRetrievalMechanismSummary, type LoCoMoRetrievalSessionReceipt, type LoCoMoRetrievalStructuralTrace, type LoCoMoRetrievalTaskDelta, type LoCoMoRetrievalTaskReceipt, type LoCoMoRetrievalTraceCoreCaptureReceipt, type LoCoMoRetrievalTraceDeltaReport, type LoCoMoRetrievalTraceProfile, type LoCoMoRetrievalTraceReceipt, type LoCoMoRetrievalTraceSelectionManifest, type LoCoMoRetrievalTraceSelector, type LoCoMoStructuralMultisetDelta, type LoComoCategoryDelta, type LoComoFinalContextRegression, type LoComoMetricDelta, type LoComoProfileArtifactEvidence, type LoComoProfileDeltaReport, type LoComoRawResultEvidence, type LoComoRecallCategoryDelta, type LoComoRecallContextSummary, type LoComoRecallDeltaReport, type LoComoRecallLineDelta, type LoComoRecallLineEvidence, type LoComoRecallMetricDelta, type LoComoRecallResultProvenance, type LoComoRecallTextDigest, type LoComoTaskRegression, type LoadDatasetOptions, type LoadSealedQrelsOptions, type LoadedDataset, type LoadedJudgeCalibrationState, type LocalLabManifest, type LocalLabManifestNotes, type LocalLabPhase, type LocalLabPhaseDescriptor, type LocalLabPhaseExecute, type LocalLabPhaseName, type LocalLabPhaseOutcome, LocalLabPreflightError, type LocalLabPreflightFailure, type LocalLabPreflightInput, type LocalLabPreflightOptions, type LocalLabPreflightResult, type LocalLabPreflightSuccess, type LocalLabProviderKind, type LocalLabRoleConfig, type LocalLlmProviderConfig, MAX_ROW_ATTEMPTS, MEMCORRECT_CORRECTION_ACCEPTANCE_RUBRIC, MEMCORRECT_CORRECTION_ACCEPTANCE_RUBRIC_VERSION, MEMCORRECT_STALE_HARM_RUBRIC, MEMCORRECT_STALE_HARM_RUBRIC_VERSION, MEMORY_EVAL_DIMENSIONS, MEMORY_EVAL_PUBLIC_LINE, MIN_CALIBRATION_SOURCE_TASKS, MITIGATED_BASELINE_SCENARIOS, type MaterializeOptions, type MaterializedRepo, type McpArgumentSemantic, type McpBackendErrorCode, type McpBackendResult, type McpBenchMemoryAdapter, type McpConformanceResult, type McpHttpTransportConfig, type McpListedTool, type McpMemCorrectAdapter, type McpMemoryAdapterOptions, McpMemoryBackendError, type McpMemoryToolMapping, type McpMemoryTransportConfig, type McpStdioTransportConfig, type McpToolCallResult, type McpToolClient, type McpToolMappingEntry, type McpToolMappingValue, type McpToolOperation, type Mem0AdapterConfig, Mem0MemCorrectAdapter, type MemCorrectGeneratorOptions, type MemCorrectJudgeRequest, type MemCorrectJudgeResult, type MemCorrectSystemAdapter, type MemoryEvalCategory, type MemoryEvalDimension, type MemoryEvalDimensionId, type MemoryEvalMetric, type MemoryGraph, type MemoryStats, type MemorySystem, type Message, type MetricAggregate, type MicroMetric, MissingCredentialError, type MitigatedBaselineConfig, type MitigatedTargetConfig, type MultipleChoiceQuestion, OPENAI_RESPONSES_JUDGE_RUBRIC_VERSION, OTHER_NAMESPACE_MEMORIES, type OllamaChatMessage, type OllamaProviderConfig, type OpenAiCompatibleProviderConfig, OpenAiResponsesJudgeError, type OpenAiResponsesJudgeErrorCode, type OpenAiResponsesJudgeTelemetry, OpenAiResponsesProvider, type OpenAiResponsesProviderConfig, type OpenAiResponsesVerdict, type OpenAiResponsesVerdictResult, PROCEDURAL_REAL_SCENARIOS, PROCEDURAL_REAL_SCENARIOS_SMOKE, PUBLISHED_BENCHMARK_ARTIFACT_IDS, type PairedAnswerReplayCache, type PairedAnswerReplayEntry, type ParsedOllamaChatResponse, type PersonalizationRetrievalCase, type PreflightDiscoveredModel, type ProceduralAblationArtifact, type ProceduralAblationPerCase, type ProceduralAblationScenario, type ProceduralRealScenario, type ProceduralRealScenarioCategory, type ProviderBaseConfig, type ProviderConfig, type ProviderDiscoveryResult, type ProviderFactoryConfig, type PublishSkipReason, type PublishSkipRecord, type PublishedBenchmarkFeed, type PublishedBenchmarkFeedEntry, type PublishedBenchmarkId, REPEATED_FAILURE_ARMS, REPEATED_FAILURE_CONFIDENCE_LEVEL, REPEATED_FAILURE_INVALID_REASONS, REPEATED_FAILURE_STATISTICS_DRAWS, REQUIRED_FRONTMATTER_FIELDS, type RecallMetrics, type RecoveredMemory, type RegressionDetail, type RegressionGateResult$1 as RegressionGateResult, type RemnicAdapterOptions, type RepeatedFailureActionEvaluator, type RepeatedFailureArm, type RepeatedFailureCheckpointLoadResult, type RepeatedFailureCliCommandResult, type RepeatedFailureEffectAnalysis, type RepeatedFailureEpisode, type RepeatedFailureEpisodeDriver, type RepeatedFailureEpisodeEvidence, type RepeatedFailureEpisodeInput, type RepeatedFailureEpisodeRow, type RepeatedFailureExpectedDesign, type RepeatedFailureFactPairAudit, type RepeatedFailureFinalRepoEvidence, type RepeatedFailureFinalState, type RepeatedFailureGateEvent, type RepeatedFailureHolmResult, type RepeatedFailureInterval, type RepeatedFailureInvalidReason, type RepeatedFailureIsolationIdentity, type RepeatedFailureLocalToolHost, type RepeatedFailureNullableInterval, RepeatedFailureOllamaChatDriver, type RepeatedFailureOllamaChatDriverConfig, type RepeatedFailureProposedAction, type RepeatedFailureRowCheckpoint, type RepeatedFailureRowClaim, type RepeatedFailureRowIdentity, RepeatedFailureRowStore, type RepeatedFailureRowStoreOptions, type RepeatedFailureRunMetadata, type RepeatedFailureStatisticalAnalysis, type RepeatedFailureSuiteManifest, type RepeatedFailureSupportDecision, type RepeatedFailureTaskCut, type RepeatedFailureTimidityAnalysis, type RepeatedFailureTokenUsage, type RepeatedFailureTokenizer, type RepeatedFailureToolDefinition, type RepeatedFailureToolExecutionResult, type RepeatedFailureTrapAuditArtifact, type RepeatedFailureTrapAuditExpected, type RepeatedFailureTrapAuditMetrics, type RepeatedFailureTrapAuditRow, type RepeatedFailureTrapAuditRowIdentity, type RepeatedFailureTrapAuditThresholds, type RepeatedFailureTry, type ReplayRepeatedFailureStatisticsOptions, type ReportCardProvenanceContext, type ResolveBenchRuntimeProfileOptions, type ResolvedBenchRuntimeProfile, type ResolvedLocalLabProfile, type ResolvedLocalLabRole, type ResolvedRunBenchmarkOptions, type ResponsesApiOutputItem, type ResponsesApiRequest, type ResponsesApiResponse, type ResponsesApiUsage, type RetrievalMissStage, type RevisionShas, type RotatedChoices, type RunBenchmarkOptions, type RunJudgeCalibrationOptions, type RunProceduralAblationCliArgs, type RunProceduralAblationOptions, type RunRepeatedFailureCliCommandInput, type RunRepeatedFailureSuiteOptions, type RunRepeatedFailureSuiteResult, type RunSequentialPhasesOptions, type RunTrapAuditOptions, SCHEMA_TIER_FIXTURE, SCHEMA_TIER_SMOKE_FIXTURE, SEALED_PROMPT_REGISTRY, SINGLE_FLAG_ABLATION_MATRIX, STATE_CLASSIFICATIONS, SYNTHETIC_MEMORIES, type SanitizedDiagnosticProvider, type SavedBaseline, type SchemaTierCorpus, type SchemaTierFixture, type SchemaTierName, type SchemaTierPage, type SchemaTierPageFrontmatter, type SealedArtifact, type SealedJudgeDecision, type SealedJudgeInput, type SealedQrelsArtifact, type SealedQrelsHandle, type SealedRubric, type SearchResult, type SeededMemory, type SeededRandom, type SeededRng, type SequentialPhaseHooks, type SingleFlagAblationCell, type SingleFlagAblationId, type SpotCheckLogger, type StageObservation, type StageStatus, type StateClassification, type StateEvaluationResult, type StatisticalReport, type StrategyPatch, StrategyPatchSchema, type StructuredJudge, StructuredJudgeError, type StructuredJudgeErrorCode, type StructuredJudgeProvider, type StructuredJudgeTelemetry, type StructuredJudgeVerdict, type StructuredJudgeVerdictResult, type StructuredVerdictRequest, type SyntheticEdge, type SyntheticEmailIngestionAdapterOptions, type SyntheticFile, type SyntheticFileIR, SyntheticFileSchema, type SyntheticRepoConfig, type SyntheticSymbol, type SyntheticTargetOptions, TRAP_TAXONOMY, type TaskAttribution, type TaskAttributionGoldWitnessV1, type TaskAttributionRetrievalWitnessV1, type TaskAttributionWitness, type TaskAttributionWitnessRuntimeV1, type TaskAttributionWitnessV1, type TaskResult, type TaskTokenUsage, type TaskVariant, TaskVariantSchema, type TemporalRetrievalCase, type ThirdPartyAdapterConfig, type TierDetail, type TimelineEntry, type TokenUsage, type TrapFingerprintV1, TrapFingerprintV1Schema, type TrapTaxonomyItem, TrapTaxonomyItemSchema, type ValidationIssue, type ValidationReport, type WallMetric, type WriteBenchmarkArtifactResult, type ZepAdapterConfig, ZepMemCorrectAdapter, addContaminationEntry, aggregateTaskScores, analyzeRepeatedFailureRows, answerBenchmarkQuestion, applyPatchAndCommit, assertCanaryUnderFloor, assertIntegrityMetaPresent, assertPublishableIntegrity, assertSha256Hex, assertTrapDatasetPreflight, assistantMeetingPrepDefinition, assistantMorningBriefDefinition, assistantNextBestActionDefinition, assistantSynthesisDefinition, attributeGoldMemory, attributeRun, attributeTask, backlinkF1, binarizeJudgeScore, bootstrapCohensKappaConfidenceInterval, bootstrapMeanConfidenceInterval, buildAmaBenchDiagnosticMatrixArtifact, buildAmaBenchDiagnosticVariantSummary, buildAmaBenchLeaderboardRows, buildBaselineFromReport, buildBenchmarkArtifact, buildBenchmarkArtifactFilename, buildBenchmarkPublishFeed, buildBenchmarkReproManifest, buildBenchmarkRunSeeds, buildBuildWeekEvidenceReceipt, buildCodexCreditReceipt, buildDriftCorpus, buildJudgePayload, buildOracleTrajectoryRecall, buildProviderFreeLoCoMoRetrievalConfig, buildRepeatedFailureRowKey, buildSchemaTierFixture, buildSchemaTierSmokeFixture, calculateCodexBudgetUnits, calculateJaccardSimilarity, calendarFixture, canonicalJsonStringify, captureBenchmarkExecutionProvenance, captureLoCoMoRetrievalTrace, captureMachineFingerprint, chatFixture, checkCodingGraphRegression, checkDatasetContamination, checkRegression, clampScore, cohensD, compareResults, computeBenchmarkReproDatasetInventoryHash, computeBenchmarkReproManifestArtifactHash, computeCohensKappa, computeH6InventoryHash, computeH6SupportArtifactHashes, computeRevisionShas, computeSealHash, computeTrapAuditArtifactHash, computeTrapAuditMetrics, containsAnswer, createSeededRng$1 as createAdamSeededRng, createAmaBenchDiagnosticAdapter, createAnthropicProvider, createCanaryAdapter, createClaudeCliProvider, createCodexCliProvider, createSeededRng as createCodingGraphSeededRng, createControlledResponsesAgentDriver, createDeterministicSpotCheckLogger, createGatewayResponder, createLightweightAdapter, createLiteLlmProvider, createLocalLlmProvider, createMcpDemoMemCorrectAdapter, createMcpDemoMemoryAdapter, createMcpMemCorrectAdapter, createMcpMemoryAdapter, createMitigatedTarget, createOllamaProvider, createOpenAiCompatibleProvider, createOpenAiResponsesBenchJudge, createOpenAiResponsesProvider, createSeededRandom$1 as createProceduralAblationSeededRandom, createProvider, createProviderBackedAmaBenchRecommendedJudge, createProviderBackedJudge, createProviderBackedResponder, createProviderBackedStructuredJudge, createRemnicAdapter, createRepeatedFailureOllamaChatDriver, createResponderFromProvider, createSeededRandom, createSeededRng$2 as createSeededRng, createSpotCheckFileLogger, createStructuredBenchJudge, createStructuredJudgeFromProvider, createSyntheticEmailIngestionAdapter, createSyntheticTarget, createTimeoutGuardedAdapter, decideRepeatedFailureContent, decideRepeatedFailureStudy, decideRepeatedFailureTiming, defaultBenchmarkBaselineDir, defaultBenchmarkPublishPath, deleteBenchmarkResults, diagnoseLoCoMoRetrievalTraceDelta, diagnoseLoComoProfileDelta, diagnoseLoComoRecallDelta, discoverAllProviders, discoveryEndpointFor, emailFixture, entityRecall, evaluateTaskState, exactMatch, executeLocalRow, extractMetrics as extractCodingGraphMetrics, extractContentWords, extractMarkdownSectionsByTitle, f1Score, fixtureToAblationScenarios, formatHandoffNote, formatMissingDatasetError, generateDriftCorpus, generateFamilyVariants, generateH6BenchmarkDataset, generateReport, generateSuiteVariants, generateSyntheticRepo, getAblationCell, getBenchmark, getBenchmarkLowerIsBetter, getGitSha, getMemoryEvalDimension, getProviderBackedJudgePromptIdentity, getRemnicVersion, getTrapTaxonomyItem, hashBenchmarkArtifact, hashBytes, hashCanonicalJson, hashOrderedQuestionIds, hashString, holmAdjust, injectionSuiteResumeContractHash, integrityMetaIsComplete, interpretEffectSize, isAmaBenchUnknownLikeAnswer, isContaminationEntry, isContaminationManifest, isRepeatedFailureTimidityEquivalent, isSafeSyntheticPath, isSealedQrelsArtifact, isSha256Hex, isStructuredJudgeProvider, isTaskFailed, judgeMemCorrectCorrectionAcceptance, judgeMemCorrectStaleMemoryHarm, lexicalSimilarity, linkMatches, listBenchmarkBaselines, listBenchmarkResults, listBenchmarks, listMemoryEvalBenchmarkIds, listMemoryEvalDimensions, llmJudgeScore, llmJudgeScoreDetailed, loadAblationFixture, loadBaseline, loadBeamDatasetPreview, loadBenchmarkArtifact, loadBenchmarkBaseline, loadBenchmarkReportCardProvenance, loadBenchmarkResult, loadBenchmarkResultSummaries, loadCommittedH6BenchmarkDataset, loadCustomBenchmarkFile, loadJudgeCalibrationState, loadLoCoMo10, loadLocalLabManifest, loadLongMemEvalS, loadSealKeyFromEnv, loadSealedQrels, loadSealedRubric, matchEntity, materializeTaskRepo, mergeContaminationManifests, openSeal, orchestrateBenchmarkRuns, pairedDeltaConfidenceInterval, parseBenchmarkArtifact, parseCodexJsonlUsage, parseCustomBenchmark, parseLocalLabManifest, parseRepeatedFailureEpisodeRow, parseRubricResponse, parseSealedQrels, pickOne, pickStableQualifiedName, planInjectionSuiteRows, precisionAtK, preflightLoCoMoRetrievalTraceCapture, preflightLocalLabRole, projectFolderFixture, randomInt, recallAtK, reconcileCodexCreditLedger, redactBenchmarkResultSecrets, relativeRiskReduction, renderAttributionReportTable, renderBaselineMarkdown, renderBenchmarkResultExport, renderLoComoProfileDeltaMarkdown, renderLoComoRecallDeltaMarkdown, renderMemorySummaryForJudge, renderMemoryViewForAgent, replayRepeatedFailureStatistics, resolveAssistantAgent, resolveAssistantRubricId, resolveAssistantSeeds, resolveAssistantSpotCheckDir, resolveBenchRuntimeProfile, resolveBenchmarkPhaseTimeoutMs, resolveBenchmarkProgressLogging, resolveBenchmarkResultReference, resolveBenchmarkRunCount, resolveBenchmarkRunId, resolveCodexCreditBudgetConfig, resolveCommittedH6FixtureDirectory, resolveLocalLabJudgeProviderConfig, resolveLocalLabProfile, resolveLocalLabRole, resolveStructuredJudge, rotateDistractors, rougeL, runAssistantBenchmark, runAssistantMeetingPrepBenchmark, runAssistantMorningBriefBenchmark, runAssistantNextBestActionBenchmark, runAssistantSynthesisBenchmark, runAttributeCliCommand, runBaseline, runBenchSuite, runBenchmark, runCodingGraphBenchmark, runCustomBenchmarkFile, runDriftGenCliCommand, runExplain, runExtractionAttack, runInjectionSuiteCliCommand, runJudgeCalibration, runMitigatedBaseline, runProceduralAblation, runProceduralAblationCli, runRepeatedFailureCliCommand, runRepeatedFailurePaperReportCliCommand, runRepeatedFailureSuite, runSealedJudge, runSequentialPhases, runTrapAudit, runTrapAuditCliCommand, runWithinCodexCreditBudget, safeHexEqual, sanitizeBenchmarkResultForJson, sanitizeLoComoResultReference, saveBaseline, saveBenchmarkBaseline, schemaCompleteness, sealPayload, selectAmaBenchDiagnosticVariants, selectCalibrationSlice, selectFixtureVariant, serializeAttributionReport, serializeBenchmarkArtifact, serializeBuildWeekEvidenceReceipt, serializeH6FixtureJson, serializeJsonl, serializeLoCoMoRetrievalTraceDelta, serializeLoCoMoRetrievalTraceReceipt, serializeSealedQrels, shuffleTasks, shuffled, summarizeBenchmarkResult, timed, tokenizeContent, unresolvedHelperImports, validateDriftCorpus, validateH6Dataset, validateH6FixtureBundle, validateH6StateDefiningIndependence, validateOllamaChatEndpoint, verifyMatchingTrapAudit, verifyRubricDigest, verifyTrapAuditArtifact, writeBenchmarkArtifact, writeBenchmarkPublishFeed, writeBenchmarkReproManifest, writeBenchmarkResult, writeBuildWeekEvidenceReceipt, writeH6FixtureBundle, writeJudgeCalibrationState, writeLeaderboardArtifactsForResult, writeRepeatedFailurePaperArtifacts, writeRepeatedFailureRunMetadata, writeRepeatedFailureStatistics, zeroScores };
|
package/dist/index.js
CHANGED
|
@@ -34629,7 +34629,7 @@ async function runMemCorrectBenchmark(options) {
|
|
|
34629
34629
|
tasks.push(task);
|
|
34630
34630
|
options.onTaskComplete?.(task, tasks.length, scenarios.length);
|
|
34631
34631
|
}
|
|
34632
|
-
const
|
|
34632
|
+
const aggregateMetrics2 = computeMetricBundle({
|
|
34633
34633
|
log: aggregateLog,
|
|
34634
34634
|
corrections: aggregateCorrections,
|
|
34635
34635
|
antiEvents: aggregateAntiEvents,
|
|
@@ -34682,7 +34682,7 @@ async function runMemCorrectBenchmark(options) {
|
|
|
34682
34682
|
// probe logs (more robust than the per-task mean for fraction
|
|
34683
34683
|
// metrics when scenario sizes vary). `aggregateTaskScores` in
|
|
34684
34684
|
// results.aggregates is the per-task-mean view.
|
|
34685
|
-
aggregateMetrics
|
|
34685
|
+
aggregateMetrics: aggregateMetrics2
|
|
34686
34686
|
}
|
|
34687
34687
|
},
|
|
34688
34688
|
cost: {
|
|
@@ -40166,8 +40166,8 @@ var LOCOMO_CATEGORY_ORDER2 = ["single_hop", "multi_hop", "temporal", "open_domai
|
|
|
40166
40166
|
var LOCOMO_TASK_CATEGORY_PATTERN2 = /-(single_hop|multi_hop|temporal|open_domain|adversarial)$/;
|
|
40167
40167
|
var SOURCE_TURN_PATTERN = /^\[([^,\]\s]+),\s*turn\s+(\d+),\s*([^,\]]+?)(?:,\s*score\s+[^\]]+)?\]/i;
|
|
40168
40168
|
var SHA256_PATTERN = /^[a-f0-9]{64}$/;
|
|
40169
|
-
function sanitizeLoComoResultReference(
|
|
40170
|
-
const reference = basename2(
|
|
40169
|
+
function sanitizeLoComoResultReference(path43) {
|
|
40170
|
+
const reference = basename2(path43).replace(/[\u0000-\u001f\u007f`]/g, "_");
|
|
40171
40171
|
if (!reference) throw new Error("Result path must identify a file.");
|
|
40172
40172
|
return reference;
|
|
40173
40173
|
}
|
|
@@ -41725,50 +41725,50 @@ function assertMemoryIdRef(value) {
|
|
|
41725
41725
|
throw new Error("LoCoMo retrieval trace requires a valid content-free memoryIdRef.");
|
|
41726
41726
|
}
|
|
41727
41727
|
}
|
|
41728
|
-
function assertJsonConfig(value,
|
|
41728
|
+
function assertJsonConfig(value, path43 = "retrievalConfig") {
|
|
41729
41729
|
if (value === null || typeof value === "string" || typeof value === "boolean") return value;
|
|
41730
41730
|
if (typeof value === "number") {
|
|
41731
|
-
if (!Number.isFinite(value)) throw new Error(`${
|
|
41731
|
+
if (!Number.isFinite(value)) throw new Error(`${path43} must contain only finite JSON numbers.`);
|
|
41732
41732
|
return value;
|
|
41733
41733
|
}
|
|
41734
41734
|
if (Array.isArray(value)) {
|
|
41735
|
-
return value.map((entry, index) => assertJsonConfig(entry, `${
|
|
41735
|
+
return value.map((entry, index) => assertJsonConfig(entry, `${path43}[${index}]`));
|
|
41736
41736
|
}
|
|
41737
41737
|
if (!value || typeof value !== "object") {
|
|
41738
|
-
throw new Error(`${
|
|
41738
|
+
throw new Error(`${path43} must be JSON-serializable and provider-free.`);
|
|
41739
41739
|
}
|
|
41740
41740
|
const output = {};
|
|
41741
41741
|
for (const key of Object.keys(value).sort()) {
|
|
41742
41742
|
const child = value[key];
|
|
41743
41743
|
if (key === "openaiApiKey") {
|
|
41744
41744
|
if (child !== false) {
|
|
41745
|
-
throw new Error(`${
|
|
41745
|
+
throw new Error(`${path43}.${key} must be exactly false for provider-free capture.`);
|
|
41746
41746
|
}
|
|
41747
41747
|
output[key] = false;
|
|
41748
41748
|
continue;
|
|
41749
41749
|
}
|
|
41750
41750
|
if (isSecretKey(key)) {
|
|
41751
|
-
throw new Error(`${
|
|
41751
|
+
throw new Error(`${path43}.${key} contains secret-bearing configuration.`);
|
|
41752
41752
|
}
|
|
41753
41753
|
if (child === void 0) continue;
|
|
41754
41754
|
if (/^(?:gatewayConfig|gatewayAgentId|fastGatewayAgentId|internalProvider|llmProvider|llmModel)$/iu.test(key) || key === "modelSource" && child !== "plugin") {
|
|
41755
|
-
throw new Error(`${
|
|
41755
|
+
throw new Error(`${path43}.${key} is provider-capable configuration.`);
|
|
41756
41756
|
}
|
|
41757
|
-
output[key] = assertJsonConfig(child, `${
|
|
41757
|
+
output[key] = assertJsonConfig(child, `${path43}.${key}`);
|
|
41758
41758
|
}
|
|
41759
41759
|
return output;
|
|
41760
41760
|
}
|
|
41761
|
-
function sanitizeProviderFreeRetrievalConfig(value,
|
|
41761
|
+
function sanitizeProviderFreeRetrievalConfig(value, path43 = "retrievalConfig") {
|
|
41762
41762
|
if (value === null || typeof value === "string" || typeof value === "boolean") return value;
|
|
41763
41763
|
if (typeof value === "number") {
|
|
41764
|
-
if (!Number.isFinite(value)) throw new Error(`${
|
|
41764
|
+
if (!Number.isFinite(value)) throw new Error(`${path43} must contain only finite JSON numbers.`);
|
|
41765
41765
|
return value;
|
|
41766
41766
|
}
|
|
41767
41767
|
if (Array.isArray(value)) {
|
|
41768
|
-
return value.map((entry, index) => sanitizeProviderFreeRetrievalConfig(entry, `${
|
|
41768
|
+
return value.map((entry, index) => sanitizeProviderFreeRetrievalConfig(entry, `${path43}[${index}]`));
|
|
41769
41769
|
}
|
|
41770
41770
|
if (!value || typeof value !== "object") {
|
|
41771
|
-
throw new Error(`${
|
|
41771
|
+
throw new Error(`${path43} must be JSON-serializable.`);
|
|
41772
41772
|
}
|
|
41773
41773
|
const output = {};
|
|
41774
41774
|
for (const key of Object.keys(value).sort()) {
|
|
@@ -41777,11 +41777,16 @@ function sanitizeProviderFreeRetrievalConfig(value, path42 = "retrievalConfig")
|
|
|
41777
41777
|
if (isSecretKey(key) || /^(?:gatewayConfig|gatewayAgentId|fastGatewayAgentId|internalProvider|llmProvider|llmModel)$/iu.test(key) || key === "modelSource") {
|
|
41778
41778
|
continue;
|
|
41779
41779
|
}
|
|
41780
|
-
output[key] = sanitizeProviderFreeRetrievalConfig(child, `${
|
|
41780
|
+
output[key] = sanitizeProviderFreeRetrievalConfig(child, `${path43}.${key}`);
|
|
41781
41781
|
}
|
|
41782
41782
|
return output;
|
|
41783
41783
|
}
|
|
41784
41784
|
|
|
41785
|
+
// src/result-summary.ts
|
|
41786
|
+
import { existsSync as existsSync2 } from "fs";
|
|
41787
|
+
import { readdir as readdir8 } from "fs/promises";
|
|
41788
|
+
import path33 from "path";
|
|
41789
|
+
|
|
41785
41790
|
// src/integrity/sealed-qrels.ts
|
|
41786
41791
|
import { readFile as readFile19 } from "fs/promises";
|
|
41787
41792
|
function isSealedQrelsArtifact(value) {
|
|
@@ -41972,6 +41977,217 @@ function selectFixtureVariant(variants, seed) {
|
|
|
41972
41977
|
return chosen;
|
|
41973
41978
|
}
|
|
41974
41979
|
|
|
41980
|
+
// src/result-summary.ts
|
|
41981
|
+
function isRecord2(value) {
|
|
41982
|
+
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
41983
|
+
}
|
|
41984
|
+
function toFiniteNumber(value) {
|
|
41985
|
+
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
41986
|
+
}
|
|
41987
|
+
var metricPriority = [
|
|
41988
|
+
"score",
|
|
41989
|
+
"accuracy",
|
|
41990
|
+
"f1",
|
|
41991
|
+
"exact_match",
|
|
41992
|
+
"llm_judge",
|
|
41993
|
+
"semantic_similarity",
|
|
41994
|
+
"precision",
|
|
41995
|
+
"recall"
|
|
41996
|
+
];
|
|
41997
|
+
function compareStrings3(left, right) {
|
|
41998
|
+
return left.localeCompare(right);
|
|
41999
|
+
}
|
|
42000
|
+
function compareMetricNames(left, right) {
|
|
42001
|
+
const leftIndex = metricPriority.indexOf(left);
|
|
42002
|
+
const rightIndex = metricPriority.indexOf(right);
|
|
42003
|
+
if (leftIndex !== rightIndex) {
|
|
42004
|
+
if (leftIndex === -1) return 1;
|
|
42005
|
+
if (rightIndex === -1) return -1;
|
|
42006
|
+
return leftIndex - rightIndex;
|
|
42007
|
+
}
|
|
42008
|
+
return compareStrings3(left, right);
|
|
42009
|
+
}
|
|
42010
|
+
function compareTimestampedRuns(left, right) {
|
|
42011
|
+
if (left.timestamp === right.timestamp) {
|
|
42012
|
+
return compareStrings3(left.id, right.id);
|
|
42013
|
+
}
|
|
42014
|
+
return right.timestamp.localeCompare(left.timestamp);
|
|
42015
|
+
}
|
|
42016
|
+
function shortHash(value) {
|
|
42017
|
+
return isSha256Hex(value) ? value.slice(0, 12) : null;
|
|
42018
|
+
}
|
|
42019
|
+
function resolveSplit(value) {
|
|
42020
|
+
return value === "public" || value === "holdout" ? value : "unknown";
|
|
42021
|
+
}
|
|
42022
|
+
function resolveCanaryFloor(value) {
|
|
42023
|
+
if (typeof value === "number" && Number.isFinite(value) && value >= 0) {
|
|
42024
|
+
return value;
|
|
42025
|
+
}
|
|
42026
|
+
return CANARY_SCORE_FLOOR;
|
|
42027
|
+
}
|
|
42028
|
+
function computeIntegritySummary(meta) {
|
|
42029
|
+
const canaryScore = toFiniteNumber(meta.canaryScore);
|
|
42030
|
+
const canaryFloor = resolveCanaryFloor(meta.canaryFloor);
|
|
42031
|
+
const sealsPresent = isSha256Hex(meta.qrelsSealedHash) && isSha256Hex(meta.judgePromptHash) && isSha256Hex(meta.datasetHash);
|
|
42032
|
+
return {
|
|
42033
|
+
split: resolveSplit(meta.splitType),
|
|
42034
|
+
sealsPresent,
|
|
42035
|
+
canaryScore,
|
|
42036
|
+
canaryFloor,
|
|
42037
|
+
canaryUnderFloor: canaryScore === null ? null : canaryScore <= canaryFloor,
|
|
42038
|
+
qrelsSealedHashShort: shortHash(meta.qrelsSealedHash),
|
|
42039
|
+
judgePromptHashShort: shortHash(meta.judgePromptHash),
|
|
42040
|
+
datasetHashShort: shortHash(meta.datasetHash)
|
|
42041
|
+
};
|
|
42042
|
+
}
|
|
42043
|
+
function providerLabel(provider) {
|
|
42044
|
+
if (!provider) {
|
|
42045
|
+
return "unconfigured";
|
|
42046
|
+
}
|
|
42047
|
+
return `${provider.provider}/${provider.model}`;
|
|
42048
|
+
}
|
|
42049
|
+
function aggregateMetrics(result) {
|
|
42050
|
+
const statistics = result.results.statistics;
|
|
42051
|
+
const confidenceIntervals = statistics?.confidenceIntervals ?? {};
|
|
42052
|
+
const effectSizes = statistics?.effectSizes ?? {};
|
|
42053
|
+
return Object.entries(result.results.aggregates).map(([name, aggregate]) => {
|
|
42054
|
+
const interval = confidenceIntervals[name];
|
|
42055
|
+
const effect = effectSizes[name];
|
|
42056
|
+
return {
|
|
42057
|
+
name,
|
|
42058
|
+
mean: toFiniteNumber(aggregate?.mean),
|
|
42059
|
+
median: toFiniteNumber(aggregate?.median),
|
|
42060
|
+
stdDev: toFiniteNumber(aggregate?.stdDev),
|
|
42061
|
+
min: toFiniteNumber(aggregate?.min),
|
|
42062
|
+
max: toFiniteNumber(aggregate?.max),
|
|
42063
|
+
ciLower: toFiniteNumber(interval?.lower),
|
|
42064
|
+
ciUpper: toFiniteNumber(interval?.upper),
|
|
42065
|
+
ciLevel: toFiniteNumber(interval?.level),
|
|
42066
|
+
effectSize: toFiniteNumber(effect?.cohensD),
|
|
42067
|
+
effectInterpretation: typeof effect?.interpretation === "string" ? effect.interpretation : null
|
|
42068
|
+
};
|
|
42069
|
+
}).sort((left, right) => compareMetricNames(left.name, right.name));
|
|
42070
|
+
}
|
|
42071
|
+
function metricHighlights(metrics) {
|
|
42072
|
+
return metrics.filter((metric) => metric.mean !== null).slice(0, 3).map((metric) => ({ name: metric.name, mean: metric.mean }));
|
|
42073
|
+
}
|
|
42074
|
+
function assistantPerSeedScore(value) {
|
|
42075
|
+
if (!isRecord2(value)) return null;
|
|
42076
|
+
const scores = isRecord2(value.scores) ? value.scores : {};
|
|
42077
|
+
const seed = toFiniteNumber(value.seed);
|
|
42078
|
+
if (seed === null) return null;
|
|
42079
|
+
return {
|
|
42080
|
+
seed,
|
|
42081
|
+
identityAccuracy: toFiniteNumber(scores.identity_accuracy),
|
|
42082
|
+
stanceCoherence: toFiniteNumber(scores.stance_coherence),
|
|
42083
|
+
novelty: toFiniteNumber(scores.novelty),
|
|
42084
|
+
calibration: toFiniteNumber(scores.calibration),
|
|
42085
|
+
parseOk: value.parseOk === true,
|
|
42086
|
+
notes: typeof value.notes === "string" ? value.notes : "",
|
|
42087
|
+
latencyMs: toFiniteNumber(value.latencyMs)
|
|
42088
|
+
};
|
|
42089
|
+
}
|
|
42090
|
+
function assistantDetails(value) {
|
|
42091
|
+
if (!value || !Array.isArray(value.perSeedScores)) return null;
|
|
42092
|
+
const perSeedScores = value.perSeedScores.map(assistantPerSeedScore).filter((entry) => entry !== null);
|
|
42093
|
+
return {
|
|
42094
|
+
focus: typeof value.focus === "string" ? value.focus : null,
|
|
42095
|
+
rubricId: typeof value.rubricId === "string" ? value.rubricId : null,
|
|
42096
|
+
rubricSha256: typeof value.rubricSha256 === "string" ? value.rubricSha256 : null,
|
|
42097
|
+
perSeedScores,
|
|
42098
|
+
judgeParseFailures: toFiniteNumber(value.judgeParseFailures)
|
|
42099
|
+
};
|
|
42100
|
+
}
|
|
42101
|
+
function scoreEntries(scores) {
|
|
42102
|
+
return Object.entries(scores).map(([name, value]) => ({ name, value })).sort((left, right) => compareMetricNames(left.name, right.name));
|
|
42103
|
+
}
|
|
42104
|
+
function taskSummaries(result) {
|
|
42105
|
+
return result.results.tasks.map((task) => {
|
|
42106
|
+
const entries = scoreEntries(task.scores);
|
|
42107
|
+
return {
|
|
42108
|
+
taskId: task.taskId,
|
|
42109
|
+
question: task.question,
|
|
42110
|
+
expected: task.expected,
|
|
42111
|
+
actual: task.actual,
|
|
42112
|
+
latencyMs: task.latencyMs,
|
|
42113
|
+
totalTokens: task.tokens.input + task.tokens.output,
|
|
42114
|
+
primaryScore: entries[0]?.value ?? null,
|
|
42115
|
+
scoreEntries: entries,
|
|
42116
|
+
assistantDetails: assistantDetails(task.details)
|
|
42117
|
+
};
|
|
42118
|
+
}).sort((left, right) => compareStrings3(left.taskId, right.taskId));
|
|
42119
|
+
}
|
|
42120
|
+
function summarizeBenchmarkResult(result, filePath) {
|
|
42121
|
+
const metrics = aggregateMetrics(result);
|
|
42122
|
+
const tasks = taskSummaries(result);
|
|
42123
|
+
const systemProvider = providerLabel(result.config.systemProvider);
|
|
42124
|
+
const judgeProvider = providerLabel(result.config.judgeProvider);
|
|
42125
|
+
const remnicConfig = result.config.remnicConfig;
|
|
42126
|
+
const configString = (key) => typeof remnicConfig[key] === "string" ? remnicConfig[key] : null;
|
|
42127
|
+
return {
|
|
42128
|
+
id: result.meta.id,
|
|
42129
|
+
benchmark: result.meta.benchmark,
|
|
42130
|
+
benchmarkTier: result.meta.benchmarkTier,
|
|
42131
|
+
timestamp: result.meta.timestamp,
|
|
42132
|
+
mode: result.meta.mode,
|
|
42133
|
+
totalLatencyMs: result.cost.totalLatencyMs,
|
|
42134
|
+
meanQueryLatencyMs: result.cost.meanQueryLatencyMs,
|
|
42135
|
+
taskCount: tasks.length,
|
|
42136
|
+
metricHighlights: metricHighlights(metrics),
|
|
42137
|
+
primaryMetric: metrics[0]?.name ?? null,
|
|
42138
|
+
primaryScore: metrics[0]?.mean ?? null,
|
|
42139
|
+
runCount: result.meta.runCount,
|
|
42140
|
+
estimatedCostUsd: result.cost.estimatedCostUsd,
|
|
42141
|
+
totalTokens: result.cost.totalTokens,
|
|
42142
|
+
inputTokens: result.cost.inputTokens,
|
|
42143
|
+
outputTokens: result.cost.outputTokens,
|
|
42144
|
+
systemProvider,
|
|
42145
|
+
judgeProvider,
|
|
42146
|
+
providerKey: `${systemProvider}__${judgeProvider}`,
|
|
42147
|
+
adapterMode: result.config.adapterMode,
|
|
42148
|
+
aggregateMetrics: metrics,
|
|
42149
|
+
taskSummaries: tasks,
|
|
42150
|
+
integrity: computeIntegritySummary(result.meta),
|
|
42151
|
+
assistantRubricId: configString("assistantRubricId"),
|
|
42152
|
+
assistantRubricSha256: configString("assistantRubricSha256"),
|
|
42153
|
+
assistantRunId: configString("assistantRunId"),
|
|
42154
|
+
filePath
|
|
42155
|
+
};
|
|
42156
|
+
}
|
|
42157
|
+
async function loadBenchmarkResultSummaries(resultsDir) {
|
|
42158
|
+
if (!existsSync2(resultsDir)) {
|
|
42159
|
+
return {
|
|
42160
|
+
resultsDir,
|
|
42161
|
+
summaries: [],
|
|
42162
|
+
skippedFiles: []
|
|
42163
|
+
};
|
|
42164
|
+
}
|
|
42165
|
+
const entries = await readdir8(resultsDir, { withFileTypes: true });
|
|
42166
|
+
const summaries = [];
|
|
42167
|
+
const skippedFiles = [];
|
|
42168
|
+
for (const entry of entries) {
|
|
42169
|
+
if (!entry.isFile() || !entry.name.endsWith(".json")) {
|
|
42170
|
+
continue;
|
|
42171
|
+
}
|
|
42172
|
+
const filePath = path33.join(resultsDir, entry.name);
|
|
42173
|
+
try {
|
|
42174
|
+
const result = await loadBenchmarkResult(filePath);
|
|
42175
|
+
summaries.push(summarizeBenchmarkResult(result, filePath));
|
|
42176
|
+
} catch (error) {
|
|
42177
|
+
skippedFiles.push({
|
|
42178
|
+
filePath,
|
|
42179
|
+
reason: error instanceof Error ? error.message : String(error)
|
|
42180
|
+
});
|
|
42181
|
+
}
|
|
42182
|
+
}
|
|
42183
|
+
summaries.sort(compareTimestampedRuns);
|
|
42184
|
+
return {
|
|
42185
|
+
resultsDir,
|
|
42186
|
+
summaries,
|
|
42187
|
+
skippedFiles
|
|
42188
|
+
};
|
|
42189
|
+
}
|
|
42190
|
+
|
|
41975
42191
|
// src/benchmarks/custom/loader.ts
|
|
41976
42192
|
import { readFile as readFile20 } from "fs/promises";
|
|
41977
42193
|
import { parse as parseYaml } from "yaml";
|
|
@@ -42101,7 +42317,7 @@ function formatError(error) {
|
|
|
42101
42317
|
|
|
42102
42318
|
// src/benchmarks/custom/runner.ts
|
|
42103
42319
|
import { randomUUID as randomUUID34 } from "crypto";
|
|
42104
|
-
import
|
|
42320
|
+
import path34 from "path";
|
|
42105
42321
|
import { expandTildePath as expandTildePath4 } from "@remnic/core";
|
|
42106
42322
|
async function runCustomBenchmarkFile(filePath, options) {
|
|
42107
42323
|
const spec = await loadCustomBenchmarkFile(filePath);
|
|
@@ -42114,7 +42330,7 @@ async function runCustomBenchmarkFile(filePath, options) {
|
|
|
42114
42330
|
let cacheRestore;
|
|
42115
42331
|
let cacheCounters;
|
|
42116
42332
|
if (spec.scoring === "llm_judge" && runOptions.system.judge !== void 0 && !runOptions.noJudgeCache && (runOptions.judgeProvider ?? null) !== null) {
|
|
42117
|
-
const cacheDir = runOptions.judgeCacheDir ?
|
|
42333
|
+
const cacheDir = runOptions.judgeCacheDir ? path34.resolve(expandTildePath4(runOptions.judgeCacheDir)) : runOptions.outputDir ? path34.join(path34.resolve(expandTildePath4(runOptions.outputDir)), "judge-cache") : void 0;
|
|
42118
42334
|
if (cacheDir !== void 0) {
|
|
42119
42335
|
const originalJudge = runOptions.system.judge;
|
|
42120
42336
|
const wrapped = wrapJudgeWithCache({
|
|
@@ -42315,7 +42531,7 @@ async function scoreTask(scoring, options, question, actual, expected) {
|
|
|
42315
42531
|
}
|
|
42316
42532
|
}
|
|
42317
42533
|
function createCustomBenchmarkDefinition(benchmark, filePath) {
|
|
42318
|
-
const id = `custom:${slugify(
|
|
42534
|
+
const id = `custom:${slugify(path34.basename(filePath, path34.extname(filePath)) || benchmark.name)}`;
|
|
42319
42535
|
return {
|
|
42320
42536
|
id,
|
|
42321
42537
|
title: benchmark.name,
|
|
@@ -43187,7 +43403,7 @@ var chatFixture = {
|
|
|
43187
43403
|
// src/judges/calibration-slice.ts
|
|
43188
43404
|
import { createHash as createHash19, randomBytes as randomBytes2 } from "crypto";
|
|
43189
43405
|
import { chmod as chmod2, lstat as lstat6, mkdir as mkdir15, open as open2, readFile as readFile21, rename as rename5, unlink as unlink3, writeFile as writeFile14 } from "fs/promises";
|
|
43190
|
-
import
|
|
43406
|
+
import path35 from "path";
|
|
43191
43407
|
|
|
43192
43408
|
// src/judges/cohen-kappa.ts
|
|
43193
43409
|
var DEFAULT_KAPPA_BOOTSTRAP_SAMPLES = 2e3;
|
|
@@ -43487,7 +43703,7 @@ async function loadOrInitializeCheckpoint(benchmarkId, provenance, sliceQuestion
|
|
|
43487
43703
|
throw new Error("runJudgeCalibration: checkpoint ordered-question-id hash does not match the validated source.");
|
|
43488
43704
|
}
|
|
43489
43705
|
await ensurePrivateDirectory(provenance.dir);
|
|
43490
|
-
const checkpointPath =
|
|
43706
|
+
const checkpointPath = path35.join(provenance.dir, `${sanitizeCalibrationSegment(benchmarkId)}.checkpoint.json`);
|
|
43491
43707
|
const lockPath = `${checkpointPath}.lock`;
|
|
43492
43708
|
let lockHandle;
|
|
43493
43709
|
try {
|
|
@@ -43611,7 +43827,7 @@ async function writeJudgeCalibrationState(result, calibrationDir, identities, pr
|
|
|
43611
43827
|
...provenance ? provenance : {},
|
|
43612
43828
|
...identities ? identities : {}
|
|
43613
43829
|
};
|
|
43614
|
-
const filePath =
|
|
43830
|
+
const filePath = path35.join(calibrationDir, `${sanitizeCalibrationSegment(result.benchmarkId)}.json`);
|
|
43615
43831
|
const tempPath = `${filePath}.${randomBytes2(6).toString("hex")}.tmp`;
|
|
43616
43832
|
await writeFile14(tempPath, `${JSON.stringify(state, null, 2)}
|
|
43617
43833
|
`, { encoding: "utf8", mode: 384 });
|
|
@@ -43625,7 +43841,7 @@ async function writeJudgeCalibrationState(result, calibrationDir, identities, pr
|
|
|
43625
43841
|
return filePath;
|
|
43626
43842
|
}
|
|
43627
43843
|
async function loadJudgeCalibrationState(benchmarkId, calibrationDir) {
|
|
43628
|
-
const filePath =
|
|
43844
|
+
const filePath = path35.join(calibrationDir, `${sanitizeCalibrationSegment(benchmarkId)}.json`);
|
|
43629
43845
|
let raw;
|
|
43630
43846
|
try {
|
|
43631
43847
|
raw = await readFile21(filePath, "utf8");
|
|
@@ -43722,7 +43938,7 @@ function sanitizeCalibrationSegment(value) {
|
|
|
43722
43938
|
// src/benchmarks/remnic/procedural-recall/ablation.ts
|
|
43723
43939
|
import { mkdir as mkdir16, mkdtemp as mkdtemp12, rm as rm16, writeFile as writeFile15, readFile as readFile22 } from "fs/promises";
|
|
43724
43940
|
import os8 from "os";
|
|
43725
|
-
import
|
|
43941
|
+
import path36 from "path";
|
|
43726
43942
|
import {
|
|
43727
43943
|
StorageManager as StorageManager4,
|
|
43728
43944
|
parseConfig as parseConfig5,
|
|
@@ -43754,7 +43970,7 @@ async function runSide(scenarios, proceduralEnabled) {
|
|
|
43754
43970
|
const observed = [];
|
|
43755
43971
|
for (const scenario of scenarios) {
|
|
43756
43972
|
const dir = await mkdtemp12(
|
|
43757
|
-
|
|
43973
|
+
path36.join(os8.tmpdir(), "remnic-bench-proc-ablation-")
|
|
43758
43974
|
);
|
|
43759
43975
|
try {
|
|
43760
43976
|
const storage = new StorageManager4(dir);
|
|
@@ -43775,7 +43991,7 @@ ${body}`,
|
|
|
43775
43991
|
);
|
|
43776
43992
|
const config = parseConfig5({
|
|
43777
43993
|
memoryDir: dir,
|
|
43778
|
-
workspaceDir:
|
|
43994
|
+
workspaceDir: path36.join(dir, "ws"),
|
|
43779
43995
|
openaiApiKey: "bench-key",
|
|
43780
43996
|
procedural: {
|
|
43781
43997
|
enabled: proceduralEnabled,
|
|
@@ -43946,7 +44162,7 @@ async function runProceduralAblationCli(args) {
|
|
|
43946
44162
|
random: args.random,
|
|
43947
44163
|
seed: args.seed
|
|
43948
44164
|
});
|
|
43949
|
-
const outDir =
|
|
44165
|
+
const outDir = path36.dirname(path36.resolve(args.outPath));
|
|
43950
44166
|
await mkdir16(outDir, { recursive: true });
|
|
43951
44167
|
await writeFile15(args.outPath, JSON.stringify(artifact, null, 2) + "\n", "utf8");
|
|
43952
44168
|
return artifact;
|
|
@@ -45064,18 +45280,18 @@ function createMitigatedTarget(config) {
|
|
|
45064
45280
|
// src/security/injection-suite/runner.ts
|
|
45065
45281
|
import { createHash as createHash22 } from "crypto";
|
|
45066
45282
|
import { mkdir as mkdir19, readFile as readFile25, writeFile as writeFile18 } from "fs/promises";
|
|
45067
|
-
import
|
|
45283
|
+
import path39 from "path";
|
|
45068
45284
|
|
|
45069
45285
|
// src/security/injection-suite/claims.ts
|
|
45070
45286
|
import { hostname } from "os";
|
|
45071
45287
|
import { mkdir as mkdir18, readFile as readFile24, rename as rename7, rm as rm17, stat as stat4, utimes, writeFile as writeFile17 } from "fs/promises";
|
|
45072
|
-
import
|
|
45288
|
+
import path38 from "path";
|
|
45073
45289
|
import { randomUUID as randomUUID36 } from "crypto";
|
|
45074
45290
|
|
|
45075
45291
|
// src/security/injection-suite/store.ts
|
|
45076
45292
|
import { createHash as createHash20, randomUUID as randomUUID35 } from "crypto";
|
|
45077
45293
|
import { mkdir as mkdir17, readFile as readFile23, rename as rename6, writeFile as writeFile16 } from "fs/promises";
|
|
45078
|
-
import
|
|
45294
|
+
import path37 from "path";
|
|
45079
45295
|
|
|
45080
45296
|
// src/security/injection-suite/types.ts
|
|
45081
45297
|
var INJECTION_SUITE_VERSION = "h5-injection-suite-v1";
|
|
@@ -45115,11 +45331,11 @@ var InjectionSuiteRowStore = class {
|
|
|
45115
45331
|
outputDir;
|
|
45116
45332
|
checkpointsDir;
|
|
45117
45333
|
constructor(outputDir) {
|
|
45118
|
-
this.outputDir =
|
|
45119
|
-
this.checkpointsDir =
|
|
45334
|
+
this.outputDir = path37.resolve(outputDir);
|
|
45335
|
+
this.checkpointsDir = path37.join(this.outputDir, "checkpoints");
|
|
45120
45336
|
}
|
|
45121
45337
|
checkpointPath(identity) {
|
|
45122
|
-
return
|
|
45338
|
+
return path37.join(this.checkpointsDir, `${buildInjectionSuiteRowKey(identity)}.json`);
|
|
45123
45339
|
}
|
|
45124
45340
|
async load(identity) {
|
|
45125
45341
|
const rowKey = buildInjectionSuiteRowKey(identity);
|
|
@@ -45182,7 +45398,7 @@ var InjectionSuiteClaimLock = class {
|
|
|
45182
45398
|
heartbeatMs;
|
|
45183
45399
|
heartbeats = /* @__PURE__ */ new Map();
|
|
45184
45400
|
lockPath(rowKey) {
|
|
45185
|
-
return
|
|
45401
|
+
return path38.join(this.checkpointsDir, `${rowKey}.lock`);
|
|
45186
45402
|
}
|
|
45187
45403
|
async tryClaim(identity) {
|
|
45188
45404
|
const rowKey = buildInjectionSuiteRowKey(identity);
|
|
@@ -45208,7 +45424,7 @@ var InjectionSuiteClaimLock = class {
|
|
|
45208
45424
|
claimedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
45209
45425
|
};
|
|
45210
45426
|
try {
|
|
45211
|
-
await writeFile17(
|
|
45427
|
+
await writeFile17(path38.join(lockPath, "owner.json"), `${JSON.stringify(owner)}
|
|
45212
45428
|
`, {
|
|
45213
45429
|
flag: "wx"
|
|
45214
45430
|
});
|
|
@@ -45222,7 +45438,7 @@ var InjectionSuiteClaimLock = class {
|
|
|
45222
45438
|
async release(claim) {
|
|
45223
45439
|
this.stopHeartbeat(claim.lockPath);
|
|
45224
45440
|
try {
|
|
45225
|
-
const ownerPath =
|
|
45441
|
+
const ownerPath = path38.join(claim.lockPath, "owner.json");
|
|
45226
45442
|
const owner = JSON.parse(await readFile24(ownerPath, "utf8"));
|
|
45227
45443
|
if (owner.ownerToken !== claim.ownerToken) return;
|
|
45228
45444
|
await rename7(ownerPath, `${ownerPath}.released-${claim.ownerToken}`);
|
|
@@ -45238,7 +45454,7 @@ var InjectionSuiteClaimLock = class {
|
|
|
45238
45454
|
await rm17(released, { recursive: true, force: true });
|
|
45239
45455
|
}
|
|
45240
45456
|
async assertOwner(claim) {
|
|
45241
|
-
const owner = JSON.parse(await readFile24(
|
|
45457
|
+
const owner = JSON.parse(await readFile24(path38.join(claim.lockPath, "owner.json"), "utf8"));
|
|
45242
45458
|
if (owner.ownerToken !== claim.ownerToken) {
|
|
45243
45459
|
throw new Error(`lost injection-suite claim ${claim.rowKey}`);
|
|
45244
45460
|
}
|
|
@@ -45246,7 +45462,7 @@ var InjectionSuiteClaimLock = class {
|
|
|
45246
45462
|
startHeartbeat(lockPath) {
|
|
45247
45463
|
this.stopHeartbeat(lockPath);
|
|
45248
45464
|
const timer = setInterval(() => {
|
|
45249
|
-
void utimes(
|
|
45465
|
+
void utimes(path38.join(lockPath, "owner.json"), /* @__PURE__ */ new Date(), /* @__PURE__ */ new Date()).catch(() => void 0);
|
|
45250
45466
|
}, this.heartbeatMs);
|
|
45251
45467
|
timer.unref?.();
|
|
45252
45468
|
this.heartbeats.set(lockPath, timer);
|
|
@@ -45257,7 +45473,7 @@ var InjectionSuiteClaimLock = class {
|
|
|
45257
45473
|
clearInterval(timer);
|
|
45258
45474
|
}
|
|
45259
45475
|
async reclaimIfExpired(lockPath) {
|
|
45260
|
-
const ownerPath =
|
|
45476
|
+
const ownerPath = path38.join(lockPath, "owner.json");
|
|
45261
45477
|
let leaseMs = this.leaseMs;
|
|
45262
45478
|
let stampMs;
|
|
45263
45479
|
try {
|
|
@@ -45540,19 +45756,19 @@ async function executeRow(identity, variant, input) {
|
|
|
45540
45756
|
}
|
|
45541
45757
|
async function readRunMetadata(outputDir) {
|
|
45542
45758
|
try {
|
|
45543
|
-
return JSON.parse(await readFile25(
|
|
45759
|
+
return JSON.parse(await readFile25(path39.join(outputDir, "run.json"), "utf8"));
|
|
45544
45760
|
} catch (error) {
|
|
45545
45761
|
if (error.code === "ENOENT") return void 0;
|
|
45546
45762
|
throw error;
|
|
45547
45763
|
}
|
|
45548
45764
|
}
|
|
45549
45765
|
async function appendEpisode(outputDir, row) {
|
|
45550
|
-
await writeFile18(
|
|
45766
|
+
await writeFile18(path39.join(outputDir, "episodes.jsonl"), `${JSON.stringify(row)}
|
|
45551
45767
|
`, { flag: "a" });
|
|
45552
45768
|
}
|
|
45553
45769
|
async function ensureEpisode(outputDir, row) {
|
|
45554
45770
|
try {
|
|
45555
|
-
const existing = await readFile25(
|
|
45771
|
+
const existing = await readFile25(path39.join(outputDir, "episodes.jsonl"), "utf8");
|
|
45556
45772
|
if (existing.includes(row.rowKey)) return;
|
|
45557
45773
|
} catch (error) {
|
|
45558
45774
|
if (error.code !== "ENOENT") throw error;
|
|
@@ -45591,7 +45807,7 @@ async function runInjectionSuiteCliCommand(input) {
|
|
|
45591
45807
|
limit: input.limit ?? null
|
|
45592
45808
|
};
|
|
45593
45809
|
try {
|
|
45594
|
-
await writeFile18(
|
|
45810
|
+
await writeFile18(path39.join(input.outputDir, "run.json"), `${JSON.stringify(metadata, null, 2)}
|
|
45595
45811
|
`, {
|
|
45596
45812
|
flag: "wx"
|
|
45597
45813
|
});
|
|
@@ -45837,7 +46053,7 @@ import { performance as performance2 } from "perf_hooks";
|
|
|
45837
46053
|
import { mkdtemp as mkdtemp13, rm as rm18 } from "fs/promises";
|
|
45838
46054
|
import { statSync } from "fs";
|
|
45839
46055
|
import { tmpdir as tmpdir7 } from "os";
|
|
45840
|
-
import
|
|
46056
|
+
import path40 from "path";
|
|
45841
46057
|
import os9 from "os";
|
|
45842
46058
|
import {
|
|
45843
46059
|
GraphStore
|
|
@@ -45934,15 +46150,15 @@ async function runCodingGraphBenchmark(config = {}) {
|
|
|
45934
46150
|
const sampleRss = () => {
|
|
45935
46151
|
peakRss = Math.max(peakRss, process.memoryUsage().rss);
|
|
45936
46152
|
};
|
|
45937
|
-
const dir = await mkdtemp13(
|
|
45938
|
-
const dbPath =
|
|
46153
|
+
const dir = await mkdtemp13(path40.join(tmpdir7(), "coding-graph-bench-"));
|
|
46154
|
+
const dbPath = path40.join(dir, "bench.sqlite");
|
|
45939
46155
|
try {
|
|
45940
46156
|
const store = await GraphStore.open({ dbPath });
|
|
45941
46157
|
try {
|
|
45942
46158
|
const FULL_INDEX_SAMPLES = 3;
|
|
45943
46159
|
const fullIndexSamples = [];
|
|
45944
46160
|
for (let s = 0; s < FULL_INDEX_SAMPLES; s++) {
|
|
45945
|
-
const sampleStore = s === 0 ? store : await GraphStore.open({ dbPath:
|
|
46161
|
+
const sampleStore = s === 0 ? store : await GraphStore.open({ dbPath: path40.join(dir, `bench-warm-${s}.sqlite`) });
|
|
45946
46162
|
const fi = await timeAsync(() => sampleStore.upsertFileBatch(storeFiles));
|
|
45947
46163
|
if (!fi.result.ok) {
|
|
45948
46164
|
if (sampleStore !== store) await sampleStore.close();
|
|
@@ -46265,7 +46481,7 @@ function buildBaselineFromReport(report, note) {
|
|
|
46265
46481
|
// src/coding-graph/repeated-failure-report.ts
|
|
46266
46482
|
import { constants } from "fs";
|
|
46267
46483
|
import { lstat as lstat7, mkdir as mkdir20, open as open3 } from "fs/promises";
|
|
46268
|
-
import
|
|
46484
|
+
import path41 from "path";
|
|
46269
46485
|
import { writeFileAtomically } from "@remnic/core/maintenance/atomic-file";
|
|
46270
46486
|
import { z as z3 } from "zod";
|
|
46271
46487
|
|
|
@@ -46893,20 +47109,20 @@ var SOURCE_ARTIFACTS = [
|
|
|
46893
47109
|
async function readArtifactLeaf(filePath) {
|
|
46894
47110
|
const leaf = await lstat7(filePath);
|
|
46895
47111
|
if (!leaf.isFile()) {
|
|
46896
|
-
throw new Error(`paper artifact leaf must be a regular file: ${
|
|
47112
|
+
throw new Error(`paper artifact leaf must be a regular file: ${path41.basename(filePath)}`);
|
|
46897
47113
|
}
|
|
46898
47114
|
let handle;
|
|
46899
47115
|
try {
|
|
46900
47116
|
handle = await open3(filePath, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
46901
47117
|
} catch (error) {
|
|
46902
47118
|
if (error.code === "ELOOP") {
|
|
46903
|
-
throw new Error(`paper artifact leaf must be a regular file: ${
|
|
47119
|
+
throw new Error(`paper artifact leaf must be a regular file: ${path41.basename(filePath)}`);
|
|
46904
47120
|
}
|
|
46905
47121
|
throw error;
|
|
46906
47122
|
}
|
|
46907
47123
|
try {
|
|
46908
47124
|
if (!(await handle.stat()).isFile()) {
|
|
46909
|
-
throw new Error(`paper artifact leaf must be a regular file: ${
|
|
47125
|
+
throw new Error(`paper artifact leaf must be a regular file: ${path41.basename(filePath)}`);
|
|
46910
47126
|
}
|
|
46911
47127
|
return await handle.readFile();
|
|
46912
47128
|
} finally {
|
|
@@ -47110,7 +47326,7 @@ async function assertArtifactMayBeWritten(filePath, content) {
|
|
|
47110
47326
|
try {
|
|
47111
47327
|
const prior = await readArtifactLeaf(filePath);
|
|
47112
47328
|
if (!prior.equals(Buffer.from(content))) {
|
|
47113
|
-
throw new Error(`paper writer refuses to overwrite changed paper artifact: ${
|
|
47329
|
+
throw new Error(`paper writer refuses to overwrite changed paper artifact: ${path41.basename(filePath)}`);
|
|
47114
47330
|
}
|
|
47115
47331
|
return false;
|
|
47116
47332
|
} catch (error) {
|
|
@@ -47119,7 +47335,7 @@ async function assertArtifactMayBeWritten(filePath, content) {
|
|
|
47119
47335
|
}
|
|
47120
47336
|
}
|
|
47121
47337
|
async function writeRepeatedFailurePaperArtifacts(options) {
|
|
47122
|
-
const runDir =
|
|
47338
|
+
const runDir = path41.resolve(options.runDir);
|
|
47123
47339
|
const reproManifest = await verifyRunManifest(runDir);
|
|
47124
47340
|
const source = await readSourceArtifacts(runDir);
|
|
47125
47341
|
const runJson = JSON.parse(source["run.json"]);
|
|
@@ -47137,7 +47353,7 @@ async function writeRepeatedFailurePaperArtifacts(options) {
|
|
|
47137
47353
|
throw new Error("paper report preregistration does not match run metadata");
|
|
47138
47354
|
}
|
|
47139
47355
|
const committedFixtureDir = await resolveCommittedH6FixtureDirectory();
|
|
47140
|
-
const committedDecisionRuleBytes = (await readArtifactLeaf(
|
|
47356
|
+
const committedDecisionRuleBytes = (await readArtifactLeaf(path41.join(committedFixtureDir, "decision-rule.json"))).toString("utf8");
|
|
47141
47357
|
if (decisionRuleBytes !== committedDecisionRuleBytes) {
|
|
47142
47358
|
throw new Error("paper report decision rule differs from the frozen committed artifact");
|
|
47143
47359
|
}
|
|
@@ -47496,7 +47712,7 @@ async function writeRepeatedFailurePaperArtifacts(options) {
|
|
|
47496
47712
|
await Promise.all(writeStates.map(async (artifact) => {
|
|
47497
47713
|
const bytes = await readArtifactLeaf(artifact.artifactPath);
|
|
47498
47714
|
if (!bytes.equals(Buffer.from(artifact.content))) {
|
|
47499
|
-
throw new Error(`paper artifact verification failed: ${
|
|
47715
|
+
throw new Error(`paper artifact verification failed: ${path41.basename(artifact.artifactPath)}`);
|
|
47500
47716
|
}
|
|
47501
47717
|
}));
|
|
47502
47718
|
const artifactPaths = writeStates.map((artifact) => artifact.artifactPath);
|
|
@@ -47516,8 +47732,8 @@ async function runRepeatedFailurePaperReportCliCommand(options) {
|
|
|
47516
47732
|
return {
|
|
47517
47733
|
exitCode: 0,
|
|
47518
47734
|
output: JSON.stringify({
|
|
47519
|
-
reportPath:
|
|
47520
|
-
manifestPath:
|
|
47735
|
+
reportPath: path41.relative(path41.resolve(options.runDir), result.reportPath),
|
|
47736
|
+
manifestPath: path41.relative(path41.resolve(options.runDir), result.manifestPath)
|
|
47521
47737
|
})
|
|
47522
47738
|
};
|
|
47523
47739
|
} catch (error) {
|
|
@@ -47527,8 +47743,8 @@ async function runRepeatedFailurePaperReportCliCommand(options) {
|
|
|
47527
47743
|
|
|
47528
47744
|
// src/attribute-cli.ts
|
|
47529
47745
|
import { QmdClient } from "@remnic/core";
|
|
47530
|
-
import { lstat as lstat8, readdir as
|
|
47531
|
-
import
|
|
47746
|
+
import { lstat as lstat8, readdir as readdir9, readFile as readFile26 } from "fs/promises";
|
|
47747
|
+
import path42 from "path";
|
|
47532
47748
|
function parseFrontmatter2(fileContent) {
|
|
47533
47749
|
const lines = fileContent.split(/\r?\n/);
|
|
47534
47750
|
if (lines.length > 0 && lines[0].trim() === "---") {
|
|
@@ -47583,7 +47799,7 @@ async function scanMemoryDir(dirPath) {
|
|
|
47583
47799
|
async function walk(currentDir, depth) {
|
|
47584
47800
|
let entries;
|
|
47585
47801
|
try {
|
|
47586
|
-
entries = await
|
|
47802
|
+
entries = await readdir9(currentDir, { withFileTypes: true });
|
|
47587
47803
|
} catch {
|
|
47588
47804
|
unreadableEntries++;
|
|
47589
47805
|
return;
|
|
@@ -47592,7 +47808,7 @@ async function scanMemoryDir(dirPath) {
|
|
|
47592
47808
|
if (entry.isSymbolicLink()) {
|
|
47593
47809
|
continue;
|
|
47594
47810
|
}
|
|
47595
|
-
const fullPath =
|
|
47811
|
+
const fullPath = path42.join(currentDir, entry.name);
|
|
47596
47812
|
try {
|
|
47597
47813
|
const stats = await lstat8(fullPath);
|
|
47598
47814
|
if (stats.isSymbolicLink()) {
|
|
@@ -47606,7 +47822,7 @@ async function scanMemoryDir(dirPath) {
|
|
|
47606
47822
|
} else if (stats.isFile() && entry.name.endsWith(".md")) {
|
|
47607
47823
|
const content = await readFile26(fullPath, "utf8");
|
|
47608
47824
|
const { id, body } = parseFrontmatter2(content);
|
|
47609
|
-
const relPath =
|
|
47825
|
+
const relPath = path42.relative(dirPath, fullPath);
|
|
47610
47826
|
memories.push({
|
|
47611
47827
|
id: id ?? relPath,
|
|
47612
47828
|
content: body.trim()
|
|
@@ -47624,24 +47840,24 @@ async function scanMemoryDir(dirPath) {
|
|
|
47624
47840
|
return memories;
|
|
47625
47841
|
}
|
|
47626
47842
|
async function resolveQmdMemory(memoryDir, collection, resultPath) {
|
|
47627
|
-
const root =
|
|
47843
|
+
const root = path42.resolve(memoryDir);
|
|
47628
47844
|
const candidates = /* @__PURE__ */ new Set();
|
|
47629
47845
|
const addCandidate = (candidate) => {
|
|
47630
|
-
const resolved =
|
|
47631
|
-
const relative =
|
|
47632
|
-
if (relative !== ".." && !relative.startsWith(`..${
|
|
47846
|
+
const resolved = path42.resolve(candidate);
|
|
47847
|
+
const relative = path42.relative(root, resolved);
|
|
47848
|
+
if (relative !== ".." && !relative.startsWith(`..${path42.sep}`) && !path42.isAbsolute(relative)) {
|
|
47633
47849
|
candidates.add(resolved);
|
|
47634
47850
|
}
|
|
47635
47851
|
};
|
|
47636
47852
|
const addRelative = (relativePath) => {
|
|
47637
47853
|
const normalized = relativePath.replace(/\\/g, "/").replace(/^\/+/, "");
|
|
47638
47854
|
if (!normalized) return;
|
|
47639
|
-
addCandidate(
|
|
47855
|
+
addCandidate(path42.join(root, normalized));
|
|
47640
47856
|
if (/^\d{4}-\d{2}-\d{2}\//.test(normalized)) {
|
|
47641
|
-
addCandidate(
|
|
47857
|
+
addCandidate(path42.join(root, "facts", normalized));
|
|
47642
47858
|
}
|
|
47643
47859
|
};
|
|
47644
|
-
if (
|
|
47860
|
+
if (path42.isAbsolute(resultPath)) {
|
|
47645
47861
|
addCandidate(resultPath);
|
|
47646
47862
|
} else {
|
|
47647
47863
|
addRelative(resultPath);
|
|
@@ -48079,6 +48295,7 @@ export {
|
|
|
48079
48295
|
loadBenchmarkBaseline,
|
|
48080
48296
|
loadBenchmarkReportCardProvenance,
|
|
48081
48297
|
loadBenchmarkResult,
|
|
48298
|
+
loadBenchmarkResultSummaries,
|
|
48082
48299
|
loadCommittedH6BenchmarkDataset,
|
|
48083
48300
|
loadCustomBenchmarkFile,
|
|
48084
48301
|
loadJudgeCalibrationState,
|
|
@@ -48186,6 +48403,7 @@ export {
|
|
|
48186
48403
|
serializeSealedQrels,
|
|
48187
48404
|
shuffleTasks,
|
|
48188
48405
|
shuffled,
|
|
48406
|
+
summarizeBenchmarkResult,
|
|
48189
48407
|
timed,
|
|
48190
48408
|
tokenizeContent,
|
|
48191
48409
|
unresolvedHelperImports,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@remnic/bench",
|
|
3
|
-
"version": "9.69.
|
|
3
|
+
"version": "9.69.34",
|
|
4
4
|
"description": "Retrieval latency ladder benchmarks + CI regression gates for @remnic/core",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -41,8 +41,8 @@
|
|
|
41
41
|
"hyparquet": "^1.25.7",
|
|
42
42
|
"yaml": "^2.4.2",
|
|
43
43
|
"zod": "^3.24.0",
|
|
44
|
-
"@remnic/coding-graph": "^9.69.
|
|
45
|
-
"@remnic/core": "^9.69.
|
|
44
|
+
"@remnic/coding-graph": "^9.69.34",
|
|
45
|
+
"@remnic/core": "^9.69.34"
|
|
46
46
|
},
|
|
47
47
|
"devDependencies": {
|
|
48
48
|
"tsup": "^8.5.1",
|