@remnic/bench 9.6.32 → 9.6.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +146 -16
- package/dist/index.js +493 -53
- package/package.json +3 -3
package/dist/index.d.ts
CHANGED
|
@@ -2152,6 +2152,7 @@ declare function createCodexCliProvider(config: CodexCliProviderConfig, deps?: C
|
|
|
2152
2152
|
declare function redactBenchmarkResultSecrets<T>(value: T): T;
|
|
2153
2153
|
declare function writeBenchmarkResult(result: BenchmarkResult, outputDir: string): Promise<string>;
|
|
2154
2154
|
declare function getRemnicVersion(): Promise<string>;
|
|
2155
|
+
declare function getGitSha(): string;
|
|
2155
2156
|
|
|
2156
2157
|
interface DiscoverAllProvidersOptions {
|
|
2157
2158
|
includeCodexCli?: boolean;
|
|
@@ -3322,6 +3323,148 @@ declare function sanitizeLoComoResultReference(path: string): string;
|
|
|
3322
3323
|
declare function diagnoseLoComoRecallDelta(options: DiagnoseLoComoRecallDeltaOptions): LoComoRecallDeltaReport;
|
|
3323
3324
|
declare function renderLoComoRecallDeltaMarkdown(report: LoComoRecallDeltaReport): string;
|
|
3324
3325
|
|
|
3326
|
+
interface LoCoMoQA {
|
|
3327
|
+
question: string;
|
|
3328
|
+
answer: string;
|
|
3329
|
+
evidence: string[];
|
|
3330
|
+
category: number;
|
|
3331
|
+
}
|
|
3332
|
+
interface LoCoMoConversation {
|
|
3333
|
+
sample_id: string;
|
|
3334
|
+
conversation: Record<string, unknown>;
|
|
3335
|
+
qa: LoCoMoQA[];
|
|
3336
|
+
event_summary?: unknown;
|
|
3337
|
+
observation?: unknown;
|
|
3338
|
+
session_summary?: unknown;
|
|
3339
|
+
}
|
|
3340
|
+
|
|
3341
|
+
/**
|
|
3342
|
+
* LoCoMo runner migrated into @remnic/bench for phase 1.
|
|
3343
|
+
*
|
|
3344
|
+
* As of issue #566 PR 2/7, the per-item lifecycle (reset → ingest →
|
|
3345
|
+
* recall → answer → judge → score) lives in `../harness.ts`. This
|
|
3346
|
+
* module only knows about dataset loading, session extraction, and
|
|
3347
|
+
* how to translate a `LoCoMoConversation` into a `HarnessPlan`.
|
|
3348
|
+
*/
|
|
3349
|
+
|
|
3350
|
+
interface LoCoMoContentDigest {
|
|
3351
|
+
sha256: string;
|
|
3352
|
+
charCount: number;
|
|
3353
|
+
lineCount: number;
|
|
3354
|
+
}
|
|
3355
|
+
interface LoCoMoCompositionLineReceipt {
|
|
3356
|
+
inputOrdinal: number;
|
|
3357
|
+
input: LoCoMoContentDigest;
|
|
3358
|
+
output: LoCoMoContentDigest;
|
|
3359
|
+
stage: "direct" | "linked";
|
|
3360
|
+
hop?: number;
|
|
3361
|
+
visible: boolean;
|
|
3362
|
+
outputStart: number;
|
|
3363
|
+
outputEnd: number;
|
|
3364
|
+
visibleStart: number;
|
|
3365
|
+
visibleEnd: number;
|
|
3366
|
+
}
|
|
3367
|
+
interface LoCoMoRecallCompositionReceipt {
|
|
3368
|
+
schemaVersion: 1;
|
|
3369
|
+
mode: "focused" | "fallback";
|
|
3370
|
+
multiHopRecallComposition: boolean;
|
|
3371
|
+
input: LoCoMoContentDigest;
|
|
3372
|
+
output: LoCoMoContentDigest;
|
|
3373
|
+
selectedLines: LoCoMoCompositionLineReceipt[];
|
|
3374
|
+
}
|
|
3375
|
+
|
|
3376
|
+
declare const LOCOMO_RETRIEVAL_TRACE_SCHEMA_VERSION: 1;
|
|
3377
|
+
declare const LOCOMO_RETRIEVAL_TRACE_SELECTION_VERSION: 1;
|
|
3378
|
+
declare const LOCOMO_RETRIEVAL_TRACE_BUDGET_VERSION: 1;
|
|
3379
|
+
type LoCoMoRetrievalTraceProfile = "baseline" | "real";
|
|
3380
|
+
type LoCoMoRetrievalTraceSelector = {
|
|
3381
|
+
taskIds: readonly string[];
|
|
3382
|
+
sampleSize?: never;
|
|
3383
|
+
seed?: never;
|
|
3384
|
+
} | {
|
|
3385
|
+
taskIds?: never;
|
|
3386
|
+
sampleSize: number;
|
|
3387
|
+
seed: number;
|
|
3388
|
+
};
|
|
3389
|
+
interface LoCoMoRetrievalTraceSelectionManifest {
|
|
3390
|
+
algorithm: "explicit-task-ids" | "sha256-seeded-sample";
|
|
3391
|
+
version: typeof LOCOMO_RETRIEVAL_TRACE_SELECTION_VERSION;
|
|
3392
|
+
seed?: number;
|
|
3393
|
+
candidateCount: number;
|
|
3394
|
+
selectedCount: number;
|
|
3395
|
+
selectedTaskIds: string[];
|
|
3396
|
+
selectedTaskIdsSha256: string;
|
|
3397
|
+
}
|
|
3398
|
+
interface LoCoMoRetrievalTraceCoreCaptureReceipt {
|
|
3399
|
+
budget: BenchRecallTraceCoreCapture["budget"];
|
|
3400
|
+
filters: BenchRecallTraceCoreCapture["filters"];
|
|
3401
|
+
results: Array<Pick<BenchRecallTraceCoreCapture["results"][number], "memoryIdRef" | "servedBy" | "scoreDecomposition" | "admittedBy" | "rejectedBy" | "disclosure" | "estimatedTokens">>;
|
|
3402
|
+
}
|
|
3403
|
+
interface LoCoMoRetrievalTraceSelectionReceipt extends Omit<BenchRecallTraceSelection, "summary"> {
|
|
3404
|
+
summary?: Omit<NonNullable<BenchRecallTraceSelection["summary"]>, "id">;
|
|
3405
|
+
}
|
|
3406
|
+
interface LoCoMoRetrievalStructuralTrace extends Omit<BenchRecallTrace, "coreCapture" | "selections"> {
|
|
3407
|
+
selections: LoCoMoRetrievalTraceSelectionReceipt[];
|
|
3408
|
+
coreCapture?: LoCoMoRetrievalTraceCoreCaptureReceipt;
|
|
3409
|
+
}
|
|
3410
|
+
interface LoCoMoRetrievalSessionReceipt {
|
|
3411
|
+
session: LoCoMoContentDigest;
|
|
3412
|
+
trace: LoCoMoRetrievalStructuralTrace;
|
|
3413
|
+
}
|
|
3414
|
+
interface LoCoMoRetrievalTaskReceipt {
|
|
3415
|
+
taskId: string;
|
|
3416
|
+
question: LoCoMoContentDigest;
|
|
3417
|
+
recallBudgetChars: number;
|
|
3418
|
+
sessions: LoCoMoRetrievalSessionReceipt[];
|
|
3419
|
+
composition: LoCoMoRecallCompositionReceipt;
|
|
3420
|
+
}
|
|
3421
|
+
interface LoCoMoRetrievalTraceReceipt {
|
|
3422
|
+
schemaVersion: typeof LOCOMO_RETRIEVAL_TRACE_SCHEMA_VERSION;
|
|
3423
|
+
benchmarkId: "locomo";
|
|
3424
|
+
captureKind: "retrieval-only";
|
|
3425
|
+
artifactHash: string;
|
|
3426
|
+
sensitivity: {
|
|
3427
|
+
classification: "restricted";
|
|
3428
|
+
contentEncoding: "sha256+length";
|
|
3429
|
+
containsGold: false;
|
|
3430
|
+
containsRawContent: false;
|
|
3431
|
+
};
|
|
3432
|
+
provenance: {
|
|
3433
|
+
gitSha: string;
|
|
3434
|
+
remnicVersion: string;
|
|
3435
|
+
runtimeProfile: LoCoMoRetrievalTraceProfile;
|
|
3436
|
+
adapterMode: "direct";
|
|
3437
|
+
replayExtractionMode: "skip";
|
|
3438
|
+
providerFree: true;
|
|
3439
|
+
dataset: {
|
|
3440
|
+
id: "locomo-10";
|
|
3441
|
+
sha256: string;
|
|
3442
|
+
};
|
|
3443
|
+
retrievalConfigSha256: string;
|
|
3444
|
+
recallBudget: {
|
|
3445
|
+
algorithm: "benchmarkRecallBudgetForSessionCount";
|
|
3446
|
+
version: typeof LOCOMO_RETRIEVAL_TRACE_BUDGET_VERSION;
|
|
3447
|
+
};
|
|
3448
|
+
};
|
|
3449
|
+
selection: LoCoMoRetrievalTraceSelectionManifest;
|
|
3450
|
+
tasks: LoCoMoRetrievalTaskReceipt[];
|
|
3451
|
+
}
|
|
3452
|
+
interface CaptureLoCoMoRetrievalTraceOptions {
|
|
3453
|
+
datasetDir: string;
|
|
3454
|
+
runtimeProfile: LoCoMoRetrievalTraceProfile;
|
|
3455
|
+
system: BenchMemoryAdapter;
|
|
3456
|
+
retrievalConfig: Record<string, unknown>;
|
|
3457
|
+
selector: LoCoMoRetrievalTraceSelector;
|
|
3458
|
+
gitSha: string;
|
|
3459
|
+
remnicVersion: string;
|
|
3460
|
+
multiHopRecallComposition?: boolean;
|
|
3461
|
+
providerFreeConfirmed: true;
|
|
3462
|
+
}
|
|
3463
|
+
declare function preflightLoCoMoRetrievalTraceCapture(options: Omit<CaptureLoCoMoRetrievalTraceOptions, "system">): Promise<void>;
|
|
3464
|
+
declare function buildProviderFreeLoCoMoRetrievalConfig(retrievalConfig: Record<string, unknown>): Record<string, unknown>;
|
|
3465
|
+
declare function captureLoCoMoRetrievalTrace(options: CaptureLoCoMoRetrievalTraceOptions): Promise<LoCoMoRetrievalTraceReceipt>;
|
|
3466
|
+
declare function serializeLoCoMoRetrievalTraceReceipt(receipt: LoCoMoRetrievalTraceReceipt): string;
|
|
3467
|
+
|
|
3325
3468
|
/**
|
|
3326
3469
|
* Dataset-contamination guard.
|
|
3327
3470
|
*
|
|
@@ -3507,21 +3650,6 @@ interface LongMemEvalItem {
|
|
|
3507
3650
|
answer_session_ids: string[];
|
|
3508
3651
|
}
|
|
3509
3652
|
|
|
3510
|
-
interface LoCoMoQA {
|
|
3511
|
-
question: string;
|
|
3512
|
-
answer: string;
|
|
3513
|
-
evidence: string[];
|
|
3514
|
-
category: number;
|
|
3515
|
-
}
|
|
3516
|
-
interface LoCoMoConversation {
|
|
3517
|
-
sample_id: string;
|
|
3518
|
-
conversation: Record<string, unknown>;
|
|
3519
|
-
qa: LoCoMoQA[];
|
|
3520
|
-
event_summary?: unknown;
|
|
3521
|
-
observation?: unknown;
|
|
3522
|
-
session_summary?: unknown;
|
|
3523
|
-
}
|
|
3524
|
-
|
|
3525
3653
|
/**
|
|
3526
3654
|
* Shared dataset loader helpers for the published LongMemEval + LoCoMo
|
|
3527
3655
|
* benchmark runners. Wraps the fs probe + JSON parse + fallback logic
|
|
@@ -3552,6 +3680,8 @@ interface LoadedDataset<T> {
|
|
|
3552
3680
|
source: DatasetSource;
|
|
3553
3681
|
/** Filename relative to `datasetDir` when source === "dataset". */
|
|
3554
3682
|
filename?: string;
|
|
3683
|
+
/** SHA-256 of the exact dataset file, or canonical bundled smoke fixture. */
|
|
3684
|
+
sha256?: string;
|
|
3555
3685
|
items: T[];
|
|
3556
3686
|
/** Parse/read errors encountered while probing candidate filenames. */
|
|
3557
3687
|
errors: string[];
|
|
@@ -5408,4 +5538,4 @@ declare function checkCodingGraphRegression(report: CodingGraphBenchReport, base
|
|
|
5408
5538
|
*/
|
|
5409
5539
|
declare function buildBaselineFromReport(report: CodingGraphBenchReport, note: string): CodingGraphBaseline;
|
|
5410
5540
|
|
|
5411
|
-
export { AMA_BENCH_DIAGNOSTIC_VARIANTS, ASSISTANT_AGENT_CONFIG_KEY, ASSISTANT_JUDGE_CONFIG_KEY, ASSISTANT_MEETING_PREP_SCENARIOS, ASSISTANT_MEETING_PREP_SMOKE_SCENARIOS, ASSISTANT_MORNING_BRIEF_SCENARIOS, ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SMOKE_SCENARIOS, ASSISTANT_RUBRIC_DIMENSIONS, ASSISTANT_RUBRIC_ID_KEY, ASSISTANT_SEEDS_CONFIG_KEY, ASSISTANT_SPOT_CHECK_DIR_KEY, ASSISTANT_SYNTHESIS_SCENARIOS, ASSISTANT_SYNTHESIS_SMOKE_SCENARIOS, type AblationConfigOverrides, type AbstentionRetrievalCase, type AggregateMetrics, type AmaBenchDiagnosticAdapterOptions, type AmaBenchDiagnosticAnswererMode, type AmaBenchDiagnosticBreakdown, type AmaBenchDiagnosticMatrixArtifact, type AmaBenchDiagnosticRecallMode, type AmaBenchDiagnosticRunContext, type AmaBenchDiagnosticTaskEvidence, type AmaBenchDiagnosticTaskRow, type AmaBenchDiagnosticVariant, type AmaBenchDiagnosticVariantSummary, type AnthropicProviderConfig, type AssistantAgent, type AssistantMemoryFact, type AssistantMemoryGraph, type AssistantRubricDimension, type AssistantRubricRequest, type AssistantRubricScores, type AssistantRunnerOptions, type AssistantScenario, type AssistantStance, type AttackRecallOptions, type AttackRetrievalHit, type AttackerMode, BENCHMARK_ARTIFACT_SCHEMA_VERSION, BENCHMARK_INTEGRITY_META_SCHEMA, BENCHMARK_REPRO_MANIFEST_FILENAME, BENCHMARK_REPRO_MANIFEST_SCHEMA_VERSION, BENCHMARK_RESULT_SCHEMA, BENCHMARK_SPLIT_TYPES, type BaselineRow, type BaselineScenario, type BeamDatasetPreview, type BenchConfig, type BenchJudge, type BenchJudgeResult, type BenchMemoryAdapter, type BenchModelSource, type BenchReasoningEffort, type BenchRecallLineageStatus, type BenchRecallOptions, type BenchRecallSupportAssessment, type BenchRecallSupportRequest, type BenchRecallSupportStatus, type BenchRecallTrace, type BenchRecallTraceCoreCapture, type BenchRecallTraceLcmCandidate, type BenchRecallTraceRange, type BenchRecallTraceSection, type BenchRecallTraceSelection, type BenchRecallWithTraceResult, type BenchResponder, type BenchResponse, type BenchRuntimeProfile, type BenchTier, type BenchmarkArtifact, type BenchmarkArtifactEnvironment, type BenchmarkArtifactHardware, type BenchmarkArtifactJudgeCalibration, type BenchmarkArtifactPerTaskScore, type BenchmarkArtifactSystem, type BenchmarkArtifactTier, type BenchmarkCategory, type BenchmarkDefinition, type BenchmarkIntegrityMeta, type BenchmarkMeta, type BenchmarkMode, type BenchmarkReport, type BenchmarkReproManifest, type BenchmarkReproManifestDataset, type BenchmarkReproManifestFile, type BenchmarkReproManifestResult, type BenchmarkResult, type BenchmarkSplitType, type BenchmarkStatus, type BenchmarkSuiteResult, type BenchmarkTier, type BootstrapKappaOptions, type BootstrapKappaResult, type BuildBenchmarkArtifactInput, type BuildBenchmarkPublishFeedOptions, type BuildBenchmarkReproManifestOptions, type BuiltInProvider, CALIBRATION_SLICE_SIZE, CANARY_FIXED_RECALL, CANARY_SCORE_FLOOR, DEFAULT_10K_FIXTURE as CODING_GRAPH_10K_FIXTURE, CODING_GRAPH_BENCH_SCHEMA_VERSION, DEFAULT_TOLERANCE_PERCENT as CODING_GRAPH_DEFAULT_TOLERANCE, MIN_ITERATIONS as CODING_GRAPH_MIN_ITERATIONS, DEFAULT_SMOKE_FIXTURE as CODING_GRAPH_SMOKE_FIXTURE, type CalibrationAnswer, type CalibrationVerdictPair, type CanaryAdapterOptions, type CanaryFloorCheck, type ClaudeCliProviderConfig, type CodexCliProviderConfig, type CodexCreditReceipt, type CodexCreditReceiptScope, type CodexCreditReconciliationReceipt, type CodingGraphBaseline, type CodingGraphBenchConfig, type CodingGraphBenchReport, type MachineFingerprint as CodingGraphMachineFingerprint, type CodingGraphMetricKey, type RegressionMetricDetail as CodingGraphRegressionDetail, type RegressionMetricKey as CodingGraphRegressionKey, type RegressionGateResult as CodingGraphRegressionResult, type CohenKappaResult, type ComparisonMetricDelta, type ComparisonResult, type CompletionOpts, type CompletionResult, type ConfidenceInterval, type ContaminationCheckResult, type ContaminationEntry, type ContaminationManifest, type CustomBenchmarkScoring, type CustomBenchmarkSpec, type CustomBenchmarkTask, DEFAULT_ABLATION_BENCHMARK, DEFAULT_ABLATION_BOOTSTRAP_SEED, DEFAULT_ASSISTANT_RUBRIC_ID, DEFAULT_BASELINE_SCENARIOS, DEFAULT_JUDGE_BINARIZATION_THRESHOLD, DEFAULT_KAPPA_BOOTSTRAP_SAMPLES, DEFAULT_KAPPA_CONFIDENCE_LEVEL, DEFAULT_OPENAI_RESPONSES_JUDGE_MODEL, type DatasetSource, type DiagnoseLoComoProfileDeltaOptions, type DiagnoseLoComoRecallDeltaOptions, type DiscoveredModel, EMPTY_CONTAMINATION_MANIFEST, type EffectSizeInterpretation, type EffectSizeSummary, type ExplainResult, type ExtractedEntity, type ExtractedLink, type ExtractedPage, type ExtractionAttackOptions, type ExtractionAttackResult, type ExtractionAttackTarget, type FixtureGenerator, type FixtureOutput, type FixtureVariant, GENERAL_ANSWER_JUDGE_RUBRIC, type GeneratedFile, type GeneratedRepo, type GoldEntity, type GoldEntityType, type GoldGraph, type GoldLink, type GoldPage, type HarnessRng, INTEGRITY_CIPHER_ALGORITHM, INTEGRITY_HASH_ALGORITHM, INTEGRITY_META_FIELDS, type IngestionBenchAdapter, type IngestionLog, JUDGE_CALIBRATION_KAPPA_THRESHOLD, JUDGE_CALIBRATION_PROTOCOL_VERSION, type JudgeCalibrationCheckpointProvenance, type JudgeCalibrationIdentities, type JudgeCalibrationResult, type JudgeCategory, type KappaConfidenceInterval, LOCAL_LAB_PROVIDER_KINDS, LOCOMO_DATASET_FILENAMES, LOCOMO_FULL_TASK_COUNT, LOCOMO_RECALL_DIFF_LINE_LIMIT, LOCOMO_RECALL_EXCERPT_CHARS, LONG_MEM_EVAL_DATASET_FILENAMES, type LeaderboardArtifactWrite, type LettaAdapterConfig, LettaMemCorrectAdapter, type LlmJudge, type LlmProvider, type LoComoCategoryDelta, type LoComoFinalContextRegression, type LoComoMetricDelta, type LoComoProfileArtifactEvidence, type LoComoProfileDeltaReport, type LoComoRawResultEvidence, type LoComoRecallCategoryDelta, type LoComoRecallContextSummary, type LoComoRecallDeltaReport, type LoComoRecallLineDelta, type LoComoRecallLineEvidence, type LoComoRecallMetricDelta, type LoComoRecallResultProvenance, type LoComoRecallTextDigest, type LoComoTaskRegression, type LoadDatasetOptions, type LoadSealedQrelsOptions, type LoadedDataset, type LoadedJudgeCalibrationState, type LocalLabManifest, type LocalLabManifestNotes, type LocalLabPhase, type LocalLabPhaseDescriptor, type LocalLabPhaseExecute, type LocalLabPhaseName, type LocalLabPhaseOutcome, LocalLabPreflightError, type LocalLabPreflightFailure, type LocalLabPreflightInput, type LocalLabPreflightOptions, type LocalLabPreflightResult, type LocalLabPreflightSuccess, type LocalLabProviderKind, type LocalLabRoleConfig, type LocalLlmProviderConfig, MEMCORRECT_CORRECTION_ACCEPTANCE_RUBRIC, MEMCORRECT_CORRECTION_ACCEPTANCE_RUBRIC_VERSION, MEMCORRECT_STALE_HARM_RUBRIC, MEMCORRECT_STALE_HARM_RUBRIC_VERSION, MEMORY_EVAL_DIMENSIONS, MEMORY_EVAL_PUBLIC_LINE, MIN_CALIBRATION_SOURCE_TASKS, MITIGATED_BASELINE_SCENARIOS, type McpArgumentSemantic, type McpBackendErrorCode, type McpBackendResult, type McpBenchMemoryAdapter, type McpConformanceResult, type McpHttpTransportConfig, type McpListedTool, type McpMemCorrectAdapter, type McpMemoryAdapterOptions, McpMemoryBackendError, type McpMemoryToolMapping, type McpMemoryTransportConfig, type McpStdioTransportConfig, type McpToolCallResult, type McpToolClient, type McpToolMappingEntry, type McpToolMappingValue, type McpToolOperation, type Mem0AdapterConfig, Mem0MemCorrectAdapter, type MemCorrectGeneratorOptions, type MemCorrectJudgeRequest, type MemCorrectJudgeResult, type MemCorrectSystemAdapter, type MemoryEvalCategory, type MemoryEvalDimension, type MemoryEvalDimensionId, type MemoryEvalMetric, type MemoryGraph, type MemoryStats, type MemorySystem, type Message, type MetricAggregate, type MicroMetric, MissingCredentialError, type MitigatedBaselineConfig, type MitigatedTargetConfig, type MultipleChoiceQuestion, OPENAI_RESPONSES_JUDGE_RUBRIC_VERSION, OTHER_NAMESPACE_MEMORIES, type OllamaProviderConfig, type OpenAiCompatibleProviderConfig, OpenAiResponsesJudgeError, type OpenAiResponsesJudgeErrorCode, type OpenAiResponsesJudgeTelemetry, OpenAiResponsesProvider, type OpenAiResponsesProviderConfig, type OpenAiResponsesVerdict, type OpenAiResponsesVerdictResult, PROCEDURAL_REAL_SCENARIOS, PROCEDURAL_REAL_SCENARIOS_SMOKE, PUBLISHED_BENCHMARK_ARTIFACT_IDS, type PersonalizationRetrievalCase, type PreflightDiscoveredModel, type ProceduralAblationArtifact, type ProceduralAblationPerCase, type ProceduralAblationScenario, type ProceduralRealScenario, type ProceduralRealScenarioCategory, type ProviderBaseConfig, type ProviderConfig, type ProviderDiscoveryResult, type ProviderFactoryConfig, type PublishSkipReason, type PublishSkipRecord, type PublishedBenchmarkFeed, type PublishedBenchmarkFeedEntry, type PublishedBenchmarkId, REQUIRED_FRONTMATTER_FIELDS, type RecallMetrics, type RecoveredMemory, type RegressionDetail, type RegressionGateResult$1 as RegressionGateResult, type RemnicAdapterOptions, type ReportCardProvenanceContext, type ResolveBenchRuntimeProfileOptions, type ResolvedBenchRuntimeProfile, type ResolvedLocalLabProfile, type ResolvedLocalLabRole, type ResolvedRunBenchmarkOptions, type RotatedChoices, type RunBenchmarkOptions, type RunJudgeCalibrationOptions, type RunProceduralAblationCliArgs, type RunProceduralAblationOptions, type RunSequentialPhasesOptions, SCHEMA_TIER_FIXTURE, SCHEMA_TIER_SMOKE_FIXTURE, SEALED_PROMPT_REGISTRY, SINGLE_FLAG_ABLATION_MATRIX, SYNTHETIC_MEMORIES, type SanitizedDiagnosticProvider, type SavedBaseline, type SchemaTierCorpus, type SchemaTierFixture, type SchemaTierName, type SchemaTierPage, type SchemaTierPageFrontmatter, type SealedArtifact, type SealedJudgeDecision, type SealedJudgeInput, type SealedQrelsArtifact, type SealedQrelsHandle, type SealedRubric, type SearchResult, type SeededMemory, type SeededRng, type SequentialPhaseHooks, type SingleFlagAblationCell, type SingleFlagAblationId, type SpotCheckLogger, type StatisticalReport, type StructuredJudge, StructuredJudgeError, type StructuredJudgeErrorCode, type StructuredJudgeProvider, type StructuredJudgeTelemetry, type StructuredJudgeVerdict, type StructuredJudgeVerdictResult, type StructuredVerdictRequest, type SyntheticEdge, type SyntheticEmailIngestionAdapterOptions, type SyntheticFileIR, type SyntheticRepoConfig, type SyntheticSymbol, type SyntheticTargetOptions, type TaskResult, type TaskTokenUsage, type TemporalRetrievalCase, type ThirdPartyAdapterConfig, type TierDetail, type TimelineEntry, type TokenUsage, type WallMetric, type WriteBenchmarkArtifactResult, type ZepAdapterConfig, ZepMemCorrectAdapter, addContaminationEntry, aggregateTaskScores, answerBenchmarkQuestion, assertCanaryUnderFloor, assertIntegrityMetaPresent, assertPublishableIntegrity, assertSha256Hex, assistantMeetingPrepDefinition, assistantMorningBriefDefinition, assistantNextBestActionDefinition, assistantSynthesisDefinition, backlinkF1, binarizeJudgeScore, bootstrapCohensKappaConfidenceInterval, bootstrapMeanConfidenceInterval, buildAmaBenchDiagnosticMatrixArtifact, buildAmaBenchDiagnosticVariantSummary, buildAmaBenchLeaderboardRows, buildBaselineFromReport, buildBenchmarkArtifact, buildBenchmarkArtifactFilename, buildBenchmarkPublishFeed, buildBenchmarkReproManifest, buildBenchmarkRunSeeds, buildCodexCreditReceipt, buildJudgePayload, buildOracleTrajectoryRecall, buildSchemaTierFixture, buildSchemaTierSmokeFixture, calendarFixture, canonicalJsonStringify, captureMachineFingerprint, chatFixture, checkCodingGraphRegression, checkDatasetContamination, checkRegression, clampScore, cohensD, compareResults, computeCohensKappa, computeSealHash, containsAnswer, createSeededRng$1 as createAdamSeededRng, createAmaBenchDiagnosticAdapter, createAnthropicProvider, createCanaryAdapter, createClaudeCliProvider, createCodexCliProvider, createSeededRng as createCodingGraphSeededRng, createDeterministicSpotCheckLogger, createGatewayResponder, createLightweightAdapter, createLiteLlmProvider, createLocalLlmProvider, createMcpDemoMemCorrectAdapter, createMcpDemoMemoryAdapter, createMcpMemCorrectAdapter, createMcpMemoryAdapter, createMitigatedTarget, createOllamaProvider, createOpenAiCompatibleProvider, createOpenAiResponsesBenchJudge, createOpenAiResponsesProvider, createSeededRandom as createProceduralAblationSeededRandom, createProvider, createProviderBackedAmaBenchRecommendedJudge, createProviderBackedJudge, createProviderBackedResponder, createProviderBackedStructuredJudge, createRemnicAdapter, createResponderFromProvider, createSeededRng$2 as createSeededRng, createSpotCheckFileLogger, createStructuredBenchJudge, createStructuredJudgeFromProvider, createSyntheticEmailIngestionAdapter, createSyntheticTarget, createTimeoutGuardedAdapter, defaultBenchmarkBaselineDir, defaultBenchmarkPublishPath, deleteBenchmarkResults, diagnoseLoComoProfileDelta, diagnoseLoComoRecallDelta, discoverAllProviders, discoveryEndpointFor, emailFixture, entityRecall, exactMatch, extractMetrics as extractCodingGraphMetrics, extractMarkdownSectionsByTitle, f1Score, fixtureToAblationScenarios, formatHandoffNote, formatMissingDatasetError, generateReport, generateSyntheticRepo, getAblationCell, getBenchmark, getBenchmarkLowerIsBetter, getMemoryEvalDimension, getProviderBackedJudgePromptIdentity, getRemnicVersion, hashBenchmarkArtifact, hashBytes, hashCanonicalJson, hashOrderedQuestionIds, hashString, integrityMetaIsComplete, interpretEffectSize, isAmaBenchUnknownLikeAnswer, isContaminationEntry, isContaminationManifest, isSealedQrelsArtifact, isSha256Hex, isStructuredJudgeProvider, judgeMemCorrectCorrectionAcceptance, judgeMemCorrectStaleMemoryHarm, linkMatches, listBenchmarkBaselines, listBenchmarkResults, listBenchmarks, listMemoryEvalBenchmarkIds, listMemoryEvalDimensions, llmJudgeScore, llmJudgeScoreDetailed, loadAblationFixture, loadBaseline, loadBeamDatasetPreview, loadBenchmarkArtifact, loadBenchmarkBaseline, loadBenchmarkReportCardProvenance, loadBenchmarkResult, loadCustomBenchmarkFile, loadJudgeCalibrationState, loadLoCoMo10, loadLocalLabManifest, loadLongMemEvalS, loadSealKeyFromEnv, loadSealedQrels, loadSealedRubric, matchEntity, mergeContaminationManifests, openSeal, orchestrateBenchmarkRuns, pairedDeltaConfidenceInterval, parseBenchmarkArtifact, parseCustomBenchmark, parseLocalLabManifest, parseRubricResponse, parseSealedQrels, pickStableQualifiedName, precisionAtK, preflightLocalLabRole, projectFolderFixture, recallAtK, reconcileCodexCreditLedger, redactBenchmarkResultSecrets, renderBaselineMarkdown, renderBenchmarkResultExport, renderLoComoProfileDeltaMarkdown, renderLoComoRecallDeltaMarkdown, renderMemorySummaryForJudge, renderMemoryViewForAgent, resolveAssistantAgent, resolveAssistantRubricId, resolveAssistantSeeds, resolveAssistantSpotCheckDir, resolveBenchRuntimeProfile, resolveBenchmarkPhaseTimeoutMs, resolveBenchmarkProgressLogging, resolveBenchmarkResultReference, resolveBenchmarkRunCount, resolveLocalLabJudgeProviderConfig, resolveLocalLabProfile, resolveLocalLabRole, resolveStructuredJudge, rotateDistractors, rougeL, runAssistantBenchmark, runAssistantMeetingPrepBenchmark, runAssistantMorningBriefBenchmark, runAssistantNextBestActionBenchmark, runAssistantSynthesisBenchmark, runBaseline, runBenchSuite, runBenchmark, runCodingGraphBenchmark, runCustomBenchmarkFile, runExplain, runExtractionAttack, runJudgeCalibration, runMitigatedBaseline, runProceduralAblation, runProceduralAblationCli, runSealedJudge, runSequentialPhases, safeHexEqual, sanitizeLoComoResultReference, saveBaseline, saveBenchmarkBaseline, schemaCompleteness, sealPayload, selectAmaBenchDiagnosticVariants, selectCalibrationSlice, selectFixtureVariant, serializeBenchmarkArtifact, serializeJsonl, serializeSealedQrels, shuffleTasks, timed, verifyRubricDigest, writeBenchmarkArtifact, writeBenchmarkPublishFeed, writeBenchmarkReproManifest, writeBenchmarkResult, writeJudgeCalibrationState, writeLeaderboardArtifactsForResult, zeroScores };
|
|
5541
|
+
export { AMA_BENCH_DIAGNOSTIC_VARIANTS, ASSISTANT_AGENT_CONFIG_KEY, ASSISTANT_JUDGE_CONFIG_KEY, ASSISTANT_MEETING_PREP_SCENARIOS, ASSISTANT_MEETING_PREP_SMOKE_SCENARIOS, ASSISTANT_MORNING_BRIEF_SCENARIOS, ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SMOKE_SCENARIOS, ASSISTANT_RUBRIC_DIMENSIONS, ASSISTANT_RUBRIC_ID_KEY, ASSISTANT_SEEDS_CONFIG_KEY, ASSISTANT_SPOT_CHECK_DIR_KEY, ASSISTANT_SYNTHESIS_SCENARIOS, ASSISTANT_SYNTHESIS_SMOKE_SCENARIOS, type AblationConfigOverrides, type AbstentionRetrievalCase, type AggregateMetrics, type AmaBenchDiagnosticAdapterOptions, type AmaBenchDiagnosticAnswererMode, type AmaBenchDiagnosticBreakdown, type AmaBenchDiagnosticMatrixArtifact, type AmaBenchDiagnosticRecallMode, type AmaBenchDiagnosticRunContext, type AmaBenchDiagnosticTaskEvidence, type AmaBenchDiagnosticTaskRow, type AmaBenchDiagnosticVariant, type AmaBenchDiagnosticVariantSummary, type AnthropicProviderConfig, type AssistantAgent, type AssistantMemoryFact, type AssistantMemoryGraph, type AssistantRubricDimension, type AssistantRubricRequest, type AssistantRubricScores, type AssistantRunnerOptions, type AssistantScenario, type AssistantStance, type AttackRecallOptions, type AttackRetrievalHit, type AttackerMode, BENCHMARK_ARTIFACT_SCHEMA_VERSION, BENCHMARK_INTEGRITY_META_SCHEMA, BENCHMARK_REPRO_MANIFEST_FILENAME, BENCHMARK_REPRO_MANIFEST_SCHEMA_VERSION, BENCHMARK_RESULT_SCHEMA, BENCHMARK_SPLIT_TYPES, type BaselineRow, type BaselineScenario, type BeamDatasetPreview, type BenchConfig, type BenchJudge, type BenchJudgeResult, type BenchMemoryAdapter, type BenchModelSource, type BenchReasoningEffort, type BenchRecallLineageStatus, type BenchRecallOptions, type BenchRecallSupportAssessment, type BenchRecallSupportRequest, type BenchRecallSupportStatus, type BenchRecallTrace, type BenchRecallTraceCoreCapture, type BenchRecallTraceLcmCandidate, type BenchRecallTraceRange, type BenchRecallTraceSection, type BenchRecallTraceSelection, type BenchRecallWithTraceResult, type BenchResponder, type BenchResponse, type BenchRuntimeProfile, type BenchTier, type BenchmarkArtifact, type BenchmarkArtifactEnvironment, type BenchmarkArtifactHardware, type BenchmarkArtifactJudgeCalibration, type BenchmarkArtifactPerTaskScore, type BenchmarkArtifactSystem, type BenchmarkArtifactTier, type BenchmarkCategory, type BenchmarkDefinition, type BenchmarkIntegrityMeta, type BenchmarkMeta, type BenchmarkMode, type BenchmarkReport, type BenchmarkReproManifest, type BenchmarkReproManifestDataset, type BenchmarkReproManifestFile, type BenchmarkReproManifestResult, type BenchmarkResult, type BenchmarkSplitType, type BenchmarkStatus, type BenchmarkSuiteResult, type BenchmarkTier, type BootstrapKappaOptions, type BootstrapKappaResult, type BuildBenchmarkArtifactInput, type BuildBenchmarkPublishFeedOptions, type BuildBenchmarkReproManifestOptions, type BuiltInProvider, CALIBRATION_SLICE_SIZE, CANARY_FIXED_RECALL, CANARY_SCORE_FLOOR, DEFAULT_10K_FIXTURE as CODING_GRAPH_10K_FIXTURE, CODING_GRAPH_BENCH_SCHEMA_VERSION, DEFAULT_TOLERANCE_PERCENT as CODING_GRAPH_DEFAULT_TOLERANCE, MIN_ITERATIONS as CODING_GRAPH_MIN_ITERATIONS, DEFAULT_SMOKE_FIXTURE as CODING_GRAPH_SMOKE_FIXTURE, type CalibrationAnswer, type CalibrationVerdictPair, type CanaryAdapterOptions, type CanaryFloorCheck, type CaptureLoCoMoRetrievalTraceOptions, type ClaudeCliProviderConfig, type CodexCliProviderConfig, type CodexCreditReceipt, type CodexCreditReceiptScope, type CodexCreditReconciliationReceipt, type CodingGraphBaseline, type CodingGraphBenchConfig, type CodingGraphBenchReport, type MachineFingerprint as CodingGraphMachineFingerprint, type CodingGraphMetricKey, type RegressionMetricDetail as CodingGraphRegressionDetail, type RegressionMetricKey as CodingGraphRegressionKey, type RegressionGateResult as CodingGraphRegressionResult, type CohenKappaResult, type ComparisonMetricDelta, type ComparisonResult, type CompletionOpts, type CompletionResult, type ConfidenceInterval, type ContaminationCheckResult, type ContaminationEntry, type ContaminationManifest, type CustomBenchmarkScoring, type CustomBenchmarkSpec, type CustomBenchmarkTask, DEFAULT_ABLATION_BENCHMARK, DEFAULT_ABLATION_BOOTSTRAP_SEED, DEFAULT_ASSISTANT_RUBRIC_ID, DEFAULT_BASELINE_SCENARIOS, DEFAULT_JUDGE_BINARIZATION_THRESHOLD, DEFAULT_KAPPA_BOOTSTRAP_SAMPLES, DEFAULT_KAPPA_CONFIDENCE_LEVEL, DEFAULT_OPENAI_RESPONSES_JUDGE_MODEL, type DatasetSource, type DiagnoseLoComoProfileDeltaOptions, type DiagnoseLoComoRecallDeltaOptions, type DiscoveredModel, EMPTY_CONTAMINATION_MANIFEST, type EffectSizeInterpretation, type EffectSizeSummary, type ExplainResult, type ExtractedEntity, type ExtractedLink, type ExtractedPage, type ExtractionAttackOptions, type ExtractionAttackResult, type ExtractionAttackTarget, type FixtureGenerator, type FixtureOutput, type FixtureVariant, GENERAL_ANSWER_JUDGE_RUBRIC, type GeneratedFile, type GeneratedRepo, type GoldEntity, type GoldEntityType, type GoldGraph, type GoldLink, type GoldPage, type HarnessRng, INTEGRITY_CIPHER_ALGORITHM, INTEGRITY_HASH_ALGORITHM, INTEGRITY_META_FIELDS, type IngestionBenchAdapter, type IngestionLog, JUDGE_CALIBRATION_KAPPA_THRESHOLD, JUDGE_CALIBRATION_PROTOCOL_VERSION, type JudgeCalibrationCheckpointProvenance, type JudgeCalibrationIdentities, type JudgeCalibrationResult, type JudgeCategory, type KappaConfidenceInterval, LOCAL_LAB_PROVIDER_KINDS, LOCOMO_DATASET_FILENAMES, LOCOMO_FULL_TASK_COUNT, LOCOMO_RECALL_DIFF_LINE_LIMIT, LOCOMO_RECALL_EXCERPT_CHARS, LOCOMO_RETRIEVAL_TRACE_BUDGET_VERSION, LOCOMO_RETRIEVAL_TRACE_SCHEMA_VERSION, LOCOMO_RETRIEVAL_TRACE_SELECTION_VERSION, LONG_MEM_EVAL_DATASET_FILENAMES, type LeaderboardArtifactWrite, type LettaAdapterConfig, LettaMemCorrectAdapter, type LlmJudge, type LlmProvider, type LoCoMoRetrievalSessionReceipt, type LoCoMoRetrievalStructuralTrace, type LoCoMoRetrievalTaskReceipt, type LoCoMoRetrievalTraceCoreCaptureReceipt, type LoCoMoRetrievalTraceProfile, type LoCoMoRetrievalTraceReceipt, type LoCoMoRetrievalTraceSelectionManifest, type LoCoMoRetrievalTraceSelector, type LoComoCategoryDelta, type LoComoFinalContextRegression, type LoComoMetricDelta, type LoComoProfileArtifactEvidence, type LoComoProfileDeltaReport, type LoComoRawResultEvidence, type LoComoRecallCategoryDelta, type LoComoRecallContextSummary, type LoComoRecallDeltaReport, type LoComoRecallLineDelta, type LoComoRecallLineEvidence, type LoComoRecallMetricDelta, type LoComoRecallResultProvenance, type LoComoRecallTextDigest, type LoComoTaskRegression, type LoadDatasetOptions, type LoadSealedQrelsOptions, type LoadedDataset, type LoadedJudgeCalibrationState, type LocalLabManifest, type LocalLabManifestNotes, type LocalLabPhase, type LocalLabPhaseDescriptor, type LocalLabPhaseExecute, type LocalLabPhaseName, type LocalLabPhaseOutcome, LocalLabPreflightError, type LocalLabPreflightFailure, type LocalLabPreflightInput, type LocalLabPreflightOptions, type LocalLabPreflightResult, type LocalLabPreflightSuccess, type LocalLabProviderKind, type LocalLabRoleConfig, type LocalLlmProviderConfig, MEMCORRECT_CORRECTION_ACCEPTANCE_RUBRIC, MEMCORRECT_CORRECTION_ACCEPTANCE_RUBRIC_VERSION, MEMCORRECT_STALE_HARM_RUBRIC, MEMCORRECT_STALE_HARM_RUBRIC_VERSION, MEMORY_EVAL_DIMENSIONS, MEMORY_EVAL_PUBLIC_LINE, MIN_CALIBRATION_SOURCE_TASKS, MITIGATED_BASELINE_SCENARIOS, type McpArgumentSemantic, type McpBackendErrorCode, type McpBackendResult, type McpBenchMemoryAdapter, type McpConformanceResult, type McpHttpTransportConfig, type McpListedTool, type McpMemCorrectAdapter, type McpMemoryAdapterOptions, McpMemoryBackendError, type McpMemoryToolMapping, type McpMemoryTransportConfig, type McpStdioTransportConfig, type McpToolCallResult, type McpToolClient, type McpToolMappingEntry, type McpToolMappingValue, type McpToolOperation, type Mem0AdapterConfig, Mem0MemCorrectAdapter, type MemCorrectGeneratorOptions, type MemCorrectJudgeRequest, type MemCorrectJudgeResult, type MemCorrectSystemAdapter, type MemoryEvalCategory, type MemoryEvalDimension, type MemoryEvalDimensionId, type MemoryEvalMetric, type MemoryGraph, type MemoryStats, type MemorySystem, type Message, type MetricAggregate, type MicroMetric, MissingCredentialError, type MitigatedBaselineConfig, type MitigatedTargetConfig, type MultipleChoiceQuestion, OPENAI_RESPONSES_JUDGE_RUBRIC_VERSION, OTHER_NAMESPACE_MEMORIES, type OllamaProviderConfig, type OpenAiCompatibleProviderConfig, OpenAiResponsesJudgeError, type OpenAiResponsesJudgeErrorCode, type OpenAiResponsesJudgeTelemetry, OpenAiResponsesProvider, type OpenAiResponsesProviderConfig, type OpenAiResponsesVerdict, type OpenAiResponsesVerdictResult, PROCEDURAL_REAL_SCENARIOS, PROCEDURAL_REAL_SCENARIOS_SMOKE, PUBLISHED_BENCHMARK_ARTIFACT_IDS, type PersonalizationRetrievalCase, type PreflightDiscoveredModel, type ProceduralAblationArtifact, type ProceduralAblationPerCase, type ProceduralAblationScenario, type ProceduralRealScenario, type ProceduralRealScenarioCategory, type ProviderBaseConfig, type ProviderConfig, type ProviderDiscoveryResult, type ProviderFactoryConfig, type PublishSkipReason, type PublishSkipRecord, type PublishedBenchmarkFeed, type PublishedBenchmarkFeedEntry, type PublishedBenchmarkId, REQUIRED_FRONTMATTER_FIELDS, type RecallMetrics, type RecoveredMemory, type RegressionDetail, type RegressionGateResult$1 as RegressionGateResult, type RemnicAdapterOptions, type ReportCardProvenanceContext, type ResolveBenchRuntimeProfileOptions, type ResolvedBenchRuntimeProfile, type ResolvedLocalLabProfile, type ResolvedLocalLabRole, type ResolvedRunBenchmarkOptions, type RotatedChoices, type RunBenchmarkOptions, type RunJudgeCalibrationOptions, type RunProceduralAblationCliArgs, type RunProceduralAblationOptions, type RunSequentialPhasesOptions, SCHEMA_TIER_FIXTURE, SCHEMA_TIER_SMOKE_FIXTURE, SEALED_PROMPT_REGISTRY, SINGLE_FLAG_ABLATION_MATRIX, SYNTHETIC_MEMORIES, type SanitizedDiagnosticProvider, type SavedBaseline, type SchemaTierCorpus, type SchemaTierFixture, type SchemaTierName, type SchemaTierPage, type SchemaTierPageFrontmatter, type SealedArtifact, type SealedJudgeDecision, type SealedJudgeInput, type SealedQrelsArtifact, type SealedQrelsHandle, type SealedRubric, type SearchResult, type SeededMemory, type SeededRng, type SequentialPhaseHooks, type SingleFlagAblationCell, type SingleFlagAblationId, type SpotCheckLogger, type StatisticalReport, type StructuredJudge, StructuredJudgeError, type StructuredJudgeErrorCode, type StructuredJudgeProvider, type StructuredJudgeTelemetry, type StructuredJudgeVerdict, type StructuredJudgeVerdictResult, type StructuredVerdictRequest, type SyntheticEdge, type SyntheticEmailIngestionAdapterOptions, type SyntheticFileIR, type SyntheticRepoConfig, type SyntheticSymbol, type SyntheticTargetOptions, type TaskResult, type TaskTokenUsage, type TemporalRetrievalCase, type ThirdPartyAdapterConfig, type TierDetail, type TimelineEntry, type TokenUsage, type WallMetric, type WriteBenchmarkArtifactResult, type ZepAdapterConfig, ZepMemCorrectAdapter, addContaminationEntry, aggregateTaskScores, answerBenchmarkQuestion, assertCanaryUnderFloor, assertIntegrityMetaPresent, assertPublishableIntegrity, assertSha256Hex, assistantMeetingPrepDefinition, assistantMorningBriefDefinition, assistantNextBestActionDefinition, assistantSynthesisDefinition, backlinkF1, binarizeJudgeScore, bootstrapCohensKappaConfidenceInterval, bootstrapMeanConfidenceInterval, buildAmaBenchDiagnosticMatrixArtifact, buildAmaBenchDiagnosticVariantSummary, buildAmaBenchLeaderboardRows, buildBaselineFromReport, buildBenchmarkArtifact, buildBenchmarkArtifactFilename, buildBenchmarkPublishFeed, buildBenchmarkReproManifest, buildBenchmarkRunSeeds, buildCodexCreditReceipt, buildJudgePayload, buildOracleTrajectoryRecall, buildProviderFreeLoCoMoRetrievalConfig, buildSchemaTierFixture, buildSchemaTierSmokeFixture, calendarFixture, canonicalJsonStringify, captureLoCoMoRetrievalTrace, captureMachineFingerprint, chatFixture, checkCodingGraphRegression, checkDatasetContamination, checkRegression, clampScore, cohensD, compareResults, computeCohensKappa, computeSealHash, containsAnswer, createSeededRng$1 as createAdamSeededRng, createAmaBenchDiagnosticAdapter, createAnthropicProvider, createCanaryAdapter, createClaudeCliProvider, createCodexCliProvider, createSeededRng as createCodingGraphSeededRng, createDeterministicSpotCheckLogger, createGatewayResponder, createLightweightAdapter, createLiteLlmProvider, createLocalLlmProvider, createMcpDemoMemCorrectAdapter, createMcpDemoMemoryAdapter, createMcpMemCorrectAdapter, createMcpMemoryAdapter, createMitigatedTarget, createOllamaProvider, createOpenAiCompatibleProvider, createOpenAiResponsesBenchJudge, createOpenAiResponsesProvider, createSeededRandom as createProceduralAblationSeededRandom, createProvider, createProviderBackedAmaBenchRecommendedJudge, createProviderBackedJudge, createProviderBackedResponder, createProviderBackedStructuredJudge, createRemnicAdapter, createResponderFromProvider, createSeededRng$2 as createSeededRng, createSpotCheckFileLogger, createStructuredBenchJudge, createStructuredJudgeFromProvider, createSyntheticEmailIngestionAdapter, createSyntheticTarget, createTimeoutGuardedAdapter, defaultBenchmarkBaselineDir, defaultBenchmarkPublishPath, deleteBenchmarkResults, diagnoseLoComoProfileDelta, diagnoseLoComoRecallDelta, discoverAllProviders, discoveryEndpointFor, emailFixture, entityRecall, exactMatch, extractMetrics as extractCodingGraphMetrics, extractMarkdownSectionsByTitle, f1Score, fixtureToAblationScenarios, formatHandoffNote, formatMissingDatasetError, generateReport, generateSyntheticRepo, getAblationCell, getBenchmark, getBenchmarkLowerIsBetter, getGitSha, getMemoryEvalDimension, getProviderBackedJudgePromptIdentity, getRemnicVersion, hashBenchmarkArtifact, hashBytes, hashCanonicalJson, hashOrderedQuestionIds, hashString, integrityMetaIsComplete, interpretEffectSize, isAmaBenchUnknownLikeAnswer, isContaminationEntry, isContaminationManifest, isSealedQrelsArtifact, isSha256Hex, isStructuredJudgeProvider, judgeMemCorrectCorrectionAcceptance, judgeMemCorrectStaleMemoryHarm, linkMatches, listBenchmarkBaselines, listBenchmarkResults, listBenchmarks, listMemoryEvalBenchmarkIds, listMemoryEvalDimensions, llmJudgeScore, llmJudgeScoreDetailed, loadAblationFixture, loadBaseline, loadBeamDatasetPreview, loadBenchmarkArtifact, loadBenchmarkBaseline, loadBenchmarkReportCardProvenance, loadBenchmarkResult, loadCustomBenchmarkFile, loadJudgeCalibrationState, loadLoCoMo10, loadLocalLabManifest, loadLongMemEvalS, loadSealKeyFromEnv, loadSealedQrels, loadSealedRubric, matchEntity, mergeContaminationManifests, openSeal, orchestrateBenchmarkRuns, pairedDeltaConfidenceInterval, parseBenchmarkArtifact, parseCustomBenchmark, parseLocalLabManifest, parseRubricResponse, parseSealedQrels, pickStableQualifiedName, precisionAtK, preflightLoCoMoRetrievalTraceCapture, preflightLocalLabRole, projectFolderFixture, recallAtK, reconcileCodexCreditLedger, redactBenchmarkResultSecrets, renderBaselineMarkdown, renderBenchmarkResultExport, renderLoComoProfileDeltaMarkdown, renderLoComoRecallDeltaMarkdown, renderMemorySummaryForJudge, renderMemoryViewForAgent, resolveAssistantAgent, resolveAssistantRubricId, resolveAssistantSeeds, resolveAssistantSpotCheckDir, resolveBenchRuntimeProfile, resolveBenchmarkPhaseTimeoutMs, resolveBenchmarkProgressLogging, resolveBenchmarkResultReference, resolveBenchmarkRunCount, resolveLocalLabJudgeProviderConfig, resolveLocalLabProfile, resolveLocalLabRole, resolveStructuredJudge, rotateDistractors, rougeL, runAssistantBenchmark, runAssistantMeetingPrepBenchmark, runAssistantMorningBriefBenchmark, runAssistantNextBestActionBenchmark, runAssistantSynthesisBenchmark, runBaseline, runBenchSuite, runBenchmark, runCodingGraphBenchmark, runCustomBenchmarkFile, runExplain, runExtractionAttack, runJudgeCalibration, runMitigatedBaseline, runProceduralAblation, runProceduralAblationCli, runSealedJudge, runSequentialPhases, safeHexEqual, sanitizeLoComoResultReference, saveBaseline, saveBenchmarkBaseline, schemaCompleteness, sealPayload, selectAmaBenchDiagnosticVariants, selectCalibrationSlice, selectFixtureVariant, serializeBenchmarkArtifact, serializeJsonl, serializeLoCoMoRetrievalTraceReceipt, serializeSealedQrels, shuffleTasks, timed, verifyRubricDigest, writeBenchmarkArtifact, writeBenchmarkPublishFeed, writeBenchmarkReproManifest, writeBenchmarkResult, writeJudgeCalibrationState, writeLeaderboardArtifactsForResult, zeroScores };
|
package/dist/index.js
CHANGED
|
@@ -19847,26 +19847,6 @@ import { collectTemporalLexicalCues } from "@remnic/core";
|
|
|
19847
19847
|
import { readFile as readFile15 } from "fs/promises";
|
|
19848
19848
|
import path17 from "path";
|
|
19849
19849
|
|
|
19850
|
-
// src/benchmarks/published/longmemeval/fixture.ts
|
|
19851
|
-
var LONG_MEM_EVAL_SMOKE_FIXTURE = [
|
|
19852
|
-
{
|
|
19853
|
-
question_id: 1,
|
|
19854
|
-
question_type: "single-session-user",
|
|
19855
|
-
question: "What city does the user live in?",
|
|
19856
|
-
answer: "Paris",
|
|
19857
|
-
question_date: "2025-01-01",
|
|
19858
|
-
haystack_dates: ["2024-12-01"],
|
|
19859
|
-
haystack_session_ids: ["session-1"],
|
|
19860
|
-
haystack_sessions: [
|
|
19861
|
-
[
|
|
19862
|
-
{ role: "user", content: "I moved to Paris last year." },
|
|
19863
|
-
{ role: "assistant", content: "Paris sounds great." }
|
|
19864
|
-
]
|
|
19865
|
-
],
|
|
19866
|
-
answer_session_ids: ["session-1"]
|
|
19867
|
-
}
|
|
19868
|
-
];
|
|
19869
|
-
|
|
19870
19850
|
// src/benchmarks/published/locomo/fixture.ts
|
|
19871
19851
|
var LOCOMO_SMOKE_FIXTURE = [
|
|
19872
19852
|
{
|
|
@@ -19916,6 +19896,26 @@ var LOCOMO_SMOKE_FIXTURE = [
|
|
|
19916
19896
|
}
|
|
19917
19897
|
];
|
|
19918
19898
|
|
|
19899
|
+
// src/benchmarks/published/longmemeval/fixture.ts
|
|
19900
|
+
var LONG_MEM_EVAL_SMOKE_FIXTURE = [
|
|
19901
|
+
{
|
|
19902
|
+
question_id: 1,
|
|
19903
|
+
question_type: "single-session-user",
|
|
19904
|
+
question: "What city does the user live in?",
|
|
19905
|
+
answer: "Paris",
|
|
19906
|
+
question_date: "2025-01-01",
|
|
19907
|
+
haystack_dates: ["2024-12-01"],
|
|
19908
|
+
haystack_session_ids: ["session-1"],
|
|
19909
|
+
haystack_sessions: [
|
|
19910
|
+
[
|
|
19911
|
+
{ role: "user", content: "I moved to Paris last year." },
|
|
19912
|
+
{ role: "assistant", content: "Paris sounds great." }
|
|
19913
|
+
]
|
|
19914
|
+
],
|
|
19915
|
+
answer_session_ids: ["session-1"]
|
|
19916
|
+
}
|
|
19917
|
+
];
|
|
19918
|
+
|
|
19919
19919
|
// src/benchmarks/published/dataset-loader.ts
|
|
19920
19920
|
var LONG_MEM_EVAL_DATASET_FILENAMES = Object.freeze([
|
|
19921
19921
|
"longmemeval_oracle.json",
|
|
@@ -19963,6 +19963,7 @@ async function loadDataset4(options) {
|
|
|
19963
19963
|
return {
|
|
19964
19964
|
source: "dataset",
|
|
19965
19965
|
filename,
|
|
19966
|
+
sha256: hashString(raw),
|
|
19966
19967
|
items: applyLimit4(parsed, limit),
|
|
19967
19968
|
errors
|
|
19968
19969
|
};
|
|
@@ -19978,6 +19979,7 @@ async function loadDataset4(options) {
|
|
|
19978
19979
|
}
|
|
19979
19980
|
return {
|
|
19980
19981
|
source: "smoke",
|
|
19982
|
+
sha256: hashCanonicalJson(options.smokeFixture),
|
|
19981
19983
|
items: applyLimit4([...options.smokeFixture], limit),
|
|
19982
19984
|
errors
|
|
19983
19985
|
};
|
|
@@ -21263,11 +21265,12 @@ var locomoDefinition = {
|
|
|
21263
21265
|
}
|
|
21264
21266
|
};
|
|
21265
21267
|
async function runLoCoMoBenchmark(options) {
|
|
21266
|
-
const
|
|
21268
|
+
const loaded = await loadLoCoMoDataset(
|
|
21267
21269
|
options.mode,
|
|
21268
21270
|
options.datasetDir,
|
|
21269
21271
|
options.limit
|
|
21270
21272
|
);
|
|
21273
|
+
const conversations = loaded.items;
|
|
21271
21274
|
const trialLimit = resolveTrialLimit(options.benchmarkOptions?.trialLimit);
|
|
21272
21275
|
const multiHopRecallComposition = resolveLoCoMoBooleanOption(
|
|
21273
21276
|
options.benchmarkOptions?.multiHopRecallComposition,
|
|
@@ -21276,7 +21279,7 @@ async function runLoCoMoBenchmark(options) {
|
|
|
21276
21279
|
);
|
|
21277
21280
|
const plans = applyTrialLimit(
|
|
21278
21281
|
conversations.map(
|
|
21279
|
-
(conversation) =>
|
|
21282
|
+
(conversation) => buildLoCoMoPlan(conversation, multiHopRecallComposition)
|
|
21280
21283
|
),
|
|
21281
21284
|
trialLimit
|
|
21282
21285
|
);
|
|
@@ -21347,7 +21350,7 @@ function applyTrialLimit(plans, trialLimit) {
|
|
|
21347
21350
|
}
|
|
21348
21351
|
return limitedPlans;
|
|
21349
21352
|
}
|
|
21350
|
-
function
|
|
21353
|
+
function buildLoCoMoPlan(conversation, multiHopRecallComposition) {
|
|
21351
21354
|
const sessions = extractSessions(conversation.conversation);
|
|
21352
21355
|
const speakerA = typeof conversation.conversation.speaker_a === "string" ? conversation.conversation.speaker_a : "Speaker A";
|
|
21353
21356
|
const ingestSessions = [];
|
|
@@ -21383,9 +21386,9 @@ function buildTrial(conversationId, qa, questionIndex, sessionIds, multiHopRecal
|
|
|
21383
21386
|
expected: qa.answer,
|
|
21384
21387
|
recallSessionIds: sessionIds,
|
|
21385
21388
|
answerFormat: "short-with-specifics",
|
|
21386
|
-
recallTextTransform: ({ question, recalledText }) =>
|
|
21389
|
+
recallTextTransform: ({ question, recalledText }) => transformLoCoMoRecallText({
|
|
21387
21390
|
question,
|
|
21388
|
-
recalledText
|
|
21391
|
+
recalledText,
|
|
21389
21392
|
multiHopRecallComposition
|
|
21390
21393
|
}),
|
|
21391
21394
|
answerFallback: ({ question, recalledText }) => answerLoCoMoFromRecall(question, recalledText),
|
|
@@ -21586,10 +21589,20 @@ function sanitizeLoCoMoRecallText(args) {
|
|
|
21586
21589
|
(id) => queryVisibleIds.has(id) ? id : ""
|
|
21587
21590
|
);
|
|
21588
21591
|
}
|
|
21589
|
-
function
|
|
21590
|
-
const
|
|
21591
|
-
|
|
21592
|
-
|
|
21592
|
+
function transformLoCoMoRecallText(args) {
|
|
21593
|
+
const sanitized = sanitizeLoCoMoRecallText(args);
|
|
21594
|
+
return prioritizeLoCoMoRecallTextWithTrace({
|
|
21595
|
+
...args,
|
|
21596
|
+
recalledText: sanitized
|
|
21597
|
+
}).text;
|
|
21598
|
+
}
|
|
21599
|
+
function prioritizeLoCoMoRecallTextWithTrace(args) {
|
|
21600
|
+
const inputLines = args.recalledText.replaceAll("\r\n", "\n").replaceAll("\r", "\n").split("\n").map((line) => line.trim()).filter((line) => line.length > 0);
|
|
21601
|
+
const lines = dedupePreserveOrder(inputLines);
|
|
21602
|
+
const inputOrdinalByLine = /* @__PURE__ */ new Map();
|
|
21603
|
+
inputLines.forEach((line, index) => {
|
|
21604
|
+
if (!inputOrdinalByLine.has(line)) inputOrdinalByLine.set(line, index);
|
|
21605
|
+
});
|
|
21593
21606
|
const questionTokens = expandLoCoMoQuestionTokens(
|
|
21594
21607
|
tokenizeForLoCoMo(args.question)
|
|
21595
21608
|
);
|
|
@@ -21622,25 +21635,90 @@ function prioritizeLoCoMoRecallText(args) {
|
|
|
21622
21635
|
)
|
|
21623
21636
|
];
|
|
21624
21637
|
if (direct.length === 0 && linkedHops.length === 0) {
|
|
21625
|
-
|
|
21638
|
+
const { text: text2 } = truncateLoCoMoContext(
|
|
21626
21639
|
args.recalledText,
|
|
21627
21640
|
LOCOMO_FALLBACK_CONTEXT_MAX_CHARS
|
|
21628
21641
|
);
|
|
21642
|
+
return {
|
|
21643
|
+
text: text2,
|
|
21644
|
+
receipt: {
|
|
21645
|
+
schemaVersion: 1,
|
|
21646
|
+
mode: "fallback",
|
|
21647
|
+
multiHopRecallComposition: args.multiHopRecallComposition,
|
|
21648
|
+
input: digestLoCoMoContent(args.recalledText),
|
|
21649
|
+
output: digestLoCoMoContent(text2),
|
|
21650
|
+
selectedLines: []
|
|
21651
|
+
}
|
|
21652
|
+
};
|
|
21629
21653
|
}
|
|
21630
|
-
const sections = [
|
|
21631
|
-
|
|
21632
|
-
|
|
21633
|
-
|
|
21654
|
+
const sections = ["## LoCoMo Question-Focused Evidence"];
|
|
21655
|
+
const selectedRanges = [];
|
|
21656
|
+
const appendSelectedLine = (input, stage, hop) => {
|
|
21657
|
+
const output = truncateLoCoMoLine(input);
|
|
21658
|
+
const inputOrdinal = inputOrdinalByLine.get(input);
|
|
21659
|
+
if (inputOrdinal === void 0) {
|
|
21660
|
+
throw new Error("LoCoMo composition selected a line outside its normalized input.");
|
|
21661
|
+
}
|
|
21662
|
+
const outputStart = sections.join("\n").length + 1;
|
|
21663
|
+
sections.push(output);
|
|
21664
|
+
selectedRanges.push({
|
|
21665
|
+
input,
|
|
21666
|
+
output,
|
|
21667
|
+
inputOrdinal,
|
|
21668
|
+
stage,
|
|
21669
|
+
...hop === void 0 ? {} : { hop },
|
|
21670
|
+
outputStart,
|
|
21671
|
+
outputEnd: outputStart + output.length
|
|
21672
|
+
});
|
|
21673
|
+
};
|
|
21674
|
+
for (const entry of direct) appendSelectedLine(entry.line, "direct");
|
|
21634
21675
|
for (const hop of linkedHops) {
|
|
21635
|
-
sections.push(
|
|
21636
|
-
|
|
21637
|
-
...hop.lines.map(truncateLoCoMoLine)
|
|
21638
|
-
);
|
|
21676
|
+
sections.push(`## LoCoMo Linked Evidence (hop ${hop.hop})`);
|
|
21677
|
+
for (const line of hop.lines) appendSelectedLine(line, "linked", hop.hop);
|
|
21639
21678
|
}
|
|
21640
|
-
|
|
21679
|
+
const truncation = truncateLoCoMoContext(
|
|
21641
21680
|
sections.join("\n"),
|
|
21642
21681
|
LOCOMO_FOCUSED_CONTEXT_MAX_CHARS
|
|
21643
21682
|
);
|
|
21683
|
+
const { text, safePrefixEnd } = truncation;
|
|
21684
|
+
const selectedLines = selectedRanges.map((entry) => {
|
|
21685
|
+
const visibleStart = Math.min(entry.outputStart, safePrefixEnd);
|
|
21686
|
+
const visibleEnd = Math.min(entry.outputEnd, safePrefixEnd);
|
|
21687
|
+
const visible = visibleEnd - visibleStart === entry.output.length;
|
|
21688
|
+
return buildCompositionLineReceipt(entry, visible, visibleStart, visibleEnd);
|
|
21689
|
+
});
|
|
21690
|
+
return {
|
|
21691
|
+
text,
|
|
21692
|
+
receipt: {
|
|
21693
|
+
schemaVersion: 1,
|
|
21694
|
+
mode: "focused",
|
|
21695
|
+
multiHopRecallComposition: args.multiHopRecallComposition,
|
|
21696
|
+
input: digestLoCoMoContent(args.recalledText),
|
|
21697
|
+
output: digestLoCoMoContent(text),
|
|
21698
|
+
selectedLines
|
|
21699
|
+
}
|
|
21700
|
+
};
|
|
21701
|
+
}
|
|
21702
|
+
function buildCompositionLineReceipt(entry, visible, visibleStart, visibleEnd) {
|
|
21703
|
+
return {
|
|
21704
|
+
inputOrdinal: entry.inputOrdinal,
|
|
21705
|
+
input: digestLoCoMoContent(entry.input),
|
|
21706
|
+
output: digestLoCoMoContent(entry.output),
|
|
21707
|
+
stage: entry.stage,
|
|
21708
|
+
...entry.hop === void 0 ? {} : { hop: entry.hop },
|
|
21709
|
+
visible,
|
|
21710
|
+
outputStart: entry.outputStart,
|
|
21711
|
+
outputEnd: entry.outputEnd,
|
|
21712
|
+
visibleStart,
|
|
21713
|
+
visibleEnd
|
|
21714
|
+
};
|
|
21715
|
+
}
|
|
21716
|
+
function digestLoCoMoContent(value) {
|
|
21717
|
+
return {
|
|
21718
|
+
sha256: hashString(value),
|
|
21719
|
+
charCount: value.length,
|
|
21720
|
+
lineCount: value.length === 0 ? 0 : value.split("\n").length
|
|
21721
|
+
};
|
|
21644
21722
|
}
|
|
21645
21723
|
function composeLoCoMoLinkedEvidence(args) {
|
|
21646
21724
|
if (args.direct.length === 0 || args.remainingLineBudget <= 0) {
|
|
@@ -21898,13 +21976,16 @@ function truncateLoCoMoLine(line) {
|
|
|
21898
21976
|
}
|
|
21899
21977
|
function truncateLoCoMoContext(text, maxChars) {
|
|
21900
21978
|
if (text.length <= maxChars) {
|
|
21901
|
-
return text;
|
|
21979
|
+
return { text, safePrefixEnd: text.length };
|
|
21902
21980
|
}
|
|
21903
21981
|
const truncated = text.slice(0, maxChars);
|
|
21904
21982
|
const lastNewline = truncated.lastIndexOf("\n");
|
|
21905
21983
|
const safePrefix = lastNewline > 0 ? truncated.slice(0, lastNewline) : truncated;
|
|
21906
|
-
return
|
|
21907
|
-
|
|
21984
|
+
return {
|
|
21985
|
+
text: `${safePrefix}
|
|
21986
|
+
[LoCoMo context truncated to ${maxChars} characters]`,
|
|
21987
|
+
safePrefixEnd: safePrefix.length
|
|
21988
|
+
};
|
|
21908
21989
|
}
|
|
21909
21990
|
function countHiddenEvidenceIdsInRecall(evidence, question, recalledText) {
|
|
21910
21991
|
const queryVisibleIds = collectDialogueIds(question);
|
|
@@ -21925,7 +22006,7 @@ function collectDialogueIds(text) {
|
|
|
21925
22006
|
function escapeRegExp2(value) {
|
|
21926
22007
|
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
21927
22008
|
}
|
|
21928
|
-
async function
|
|
22009
|
+
async function loadLoCoMoDataset(mode, datasetDir, limit) {
|
|
21929
22010
|
const loaded = await loadLoCoMo10({
|
|
21930
22011
|
mode,
|
|
21931
22012
|
datasetDir,
|
|
@@ -21957,7 +22038,15 @@ async function loadDataset6(mode, datasetDir, limit) {
|
|
|
21957
22038
|
"[remnic-bench] LoCoMo falling back to smoke fixture: " + loaded.errors.join(" | ")
|
|
21958
22039
|
);
|
|
21959
22040
|
}
|
|
21960
|
-
|
|
22041
|
+
if (!loaded.sha256) {
|
|
22042
|
+
throw new Error("LoCoMo dataset loader did not provide a content hash.");
|
|
22043
|
+
}
|
|
22044
|
+
return {
|
|
22045
|
+
source: loaded.source,
|
|
22046
|
+
...loaded.filename === void 0 ? {} : { filename: loaded.filename },
|
|
22047
|
+
sha256: loaded.sha256,
|
|
22048
|
+
items: loaded.items
|
|
22049
|
+
};
|
|
21961
22050
|
}
|
|
21962
22051
|
function parseDataset2(raw, filename) {
|
|
21963
22052
|
const parsed = JSON.parse(raw);
|
|
@@ -22241,7 +22330,7 @@ async function loadBeamDatasetPreview(options) {
|
|
|
22241
22330
|
}
|
|
22242
22331
|
let dataset;
|
|
22243
22332
|
try {
|
|
22244
|
-
dataset = await
|
|
22333
|
+
dataset = await loadDataset6(
|
|
22245
22334
|
options.mode === "quick" ? "quick" : "full",
|
|
22246
22335
|
options.datasetDir,
|
|
22247
22336
|
options.limit
|
|
@@ -22280,7 +22369,7 @@ async function loadBeamDatasetPreview(options) {
|
|
|
22280
22369
|
};
|
|
22281
22370
|
}
|
|
22282
22371
|
async function runBeamBenchmark(options) {
|
|
22283
|
-
const dataset = await
|
|
22372
|
+
const dataset = await loadDataset6(options.mode, options.datasetDir, options.limit);
|
|
22284
22373
|
const tasks = [];
|
|
22285
22374
|
const taskFilter = normalizeBeamTaskFilter(
|
|
22286
22375
|
options.benchmarkOptions?.taskFilter
|
|
@@ -22467,7 +22556,7 @@ async function runBeamBenchmark(options) {
|
|
|
22467
22556
|
}
|
|
22468
22557
|
};
|
|
22469
22558
|
}
|
|
22470
|
-
async function
|
|
22559
|
+
async function loadDataset6(mode, datasetDir, limit) {
|
|
22471
22560
|
const normalizedLimit = normalizeLimit5(limit);
|
|
22472
22561
|
const ensureDatasetEntries = (entryCount) => {
|
|
22473
22562
|
if (entryCount === 0) {
|
|
@@ -23653,7 +23742,7 @@ var personaMemDefinition = {
|
|
|
23653
23742
|
}
|
|
23654
23743
|
};
|
|
23655
23744
|
async function runPersonaMemBenchmark(options) {
|
|
23656
|
-
const samples = await
|
|
23745
|
+
const samples = await loadDataset7(options.mode, options.datasetDir, options.limit);
|
|
23657
23746
|
const tasks = [];
|
|
23658
23747
|
const totalTasks = samples.length;
|
|
23659
23748
|
for (let sampleIndex = 0; sampleIndex < samples.length; sampleIndex += 1) {
|
|
@@ -23833,7 +23922,7 @@ async function runPersonaMemBenchmark(options) {
|
|
|
23833
23922
|
}
|
|
23834
23923
|
};
|
|
23835
23924
|
}
|
|
23836
|
-
async function
|
|
23925
|
+
async function loadDataset7(mode, datasetDir, limit) {
|
|
23837
23926
|
const normalizedLimit = normalizeLimit6(limit);
|
|
23838
23927
|
const ensureDatasetSamples = (samples) => {
|
|
23839
23928
|
if (samples.length === 0) {
|
|
@@ -24441,7 +24530,7 @@ var memBenchDefinition = {
|
|
|
24441
24530
|
}
|
|
24442
24531
|
};
|
|
24443
24532
|
async function runMemBenchBenchmark(options) {
|
|
24444
|
-
const dataset = await
|
|
24533
|
+
const dataset = await loadDataset8(options.mode, options.datasetDir, options.limit);
|
|
24445
24534
|
const tasks = [];
|
|
24446
24535
|
const totalTasks = dataset.length;
|
|
24447
24536
|
for (const testCase of dataset) {
|
|
@@ -24615,7 +24704,7 @@ async function runMemBenchBenchmark(options) {
|
|
|
24615
24704
|
}
|
|
24616
24705
|
};
|
|
24617
24706
|
}
|
|
24618
|
-
async function
|
|
24707
|
+
async function loadDataset8(mode, datasetDir, limit) {
|
|
24619
24708
|
const normalizedLimit = normalizeLimit7(limit);
|
|
24620
24709
|
const ensureDatasetCases = (cases) => {
|
|
24621
24710
|
if (cases.length === 0) {
|
|
@@ -25646,7 +25735,7 @@ var memoryAgentBenchDefinition = {
|
|
|
25646
25735
|
}
|
|
25647
25736
|
};
|
|
25648
25737
|
async function runMemoryAgentBenchBenchmark(options) {
|
|
25649
|
-
const rawDataset = await
|
|
25738
|
+
const rawDataset = await loadDataset9(options.mode, options.datasetDir, options.limit);
|
|
25650
25739
|
const trialLimit = resolveTrialLimit2(options.benchmarkOptions?.trialLimit);
|
|
25651
25740
|
const benchmarkOptions = trialLimit === void 0 ? options.benchmarkOptions : { ...options.benchmarkOptions ?? {}, trialLimit };
|
|
25652
25741
|
const dataset = applyTrialLimit2(rawDataset, trialLimit);
|
|
@@ -26619,7 +26708,7 @@ function decodeUrlComponentSafely(value) {
|
|
|
26619
26708
|
return value;
|
|
26620
26709
|
}
|
|
26621
26710
|
}
|
|
26622
|
-
async function
|
|
26711
|
+
async function loadDataset9(mode, datasetDir, limit) {
|
|
26623
26712
|
const normalizedLimit = normalizeLimit8(limit);
|
|
26624
26713
|
const ensureDatasetItems = (items) => {
|
|
26625
26714
|
if (items.length === 0) {
|
|
@@ -39194,6 +39283,349 @@ function formatSignedScore2(value) {
|
|
|
39194
39283
|
return `${value >= 0 ? "+" : ""}${formatScore2(value)}`;
|
|
39195
39284
|
}
|
|
39196
39285
|
|
|
39286
|
+
// src/benchmarks/published/locomo/retrieval-trace-runner.ts
|
|
39287
|
+
var LOCOMO_RETRIEVAL_TRACE_SCHEMA_VERSION = 1;
|
|
39288
|
+
var LOCOMO_RETRIEVAL_TRACE_SELECTION_VERSION = 1;
|
|
39289
|
+
var LOCOMO_RETRIEVAL_TRACE_BUDGET_VERSION = 1;
|
|
39290
|
+
async function preflightLoCoMoRetrievalTraceCapture(options) {
|
|
39291
|
+
assertCaptureOptions(options);
|
|
39292
|
+
assertProviderFreeRetrievalConfig(options.retrievalConfig);
|
|
39293
|
+
const loaded = await loadLoCoMoDataset("full", options.datasetDir);
|
|
39294
|
+
const multiHopRecallComposition = options.multiHopRecallComposition ?? true;
|
|
39295
|
+
const plans = loaded.items.map((conversation) => buildLoCoMoPlan(conversation, multiHopRecallComposition));
|
|
39296
|
+
const selectable = plans.flatMap(
|
|
39297
|
+
(plan, planIndex) => plan.trials.map((trial) => ({ taskId: trial.taskId, planIndex }))
|
|
39298
|
+
);
|
|
39299
|
+
selectLoCoMoRetrievalTraceTasks(selectable, options.selector);
|
|
39300
|
+
}
|
|
39301
|
+
function buildProviderFreeLoCoMoRetrievalConfig(retrievalConfig) {
|
|
39302
|
+
const sanitized = sanitizeProviderFreeRetrievalConfig(retrievalConfig);
|
|
39303
|
+
return assertProviderFreeRetrievalConfig({
|
|
39304
|
+
...sanitized,
|
|
39305
|
+
localLlmEnabled: false,
|
|
39306
|
+
localLlmFastEnabled: false,
|
|
39307
|
+
recallPlannerEnabled: false,
|
|
39308
|
+
embeddingFallbackEnabled: false,
|
|
39309
|
+
hostEmbeddingProviderEnabled: false,
|
|
39310
|
+
openaiApiKey: false,
|
|
39311
|
+
modelSource: "plugin"
|
|
39312
|
+
});
|
|
39313
|
+
}
|
|
39314
|
+
async function captureLoCoMoRetrievalTrace(options) {
|
|
39315
|
+
assertCaptureOptions(options);
|
|
39316
|
+
const retrievalConfig = assertProviderFreeRetrievalConfig(options.retrievalConfig);
|
|
39317
|
+
const recallWithTrace = options.system.recallWithTrace?.bind(options.system);
|
|
39318
|
+
if (!recallWithTrace) {
|
|
39319
|
+
throw new Error("LoCoMo retrieval trace capture requires system.recallWithTrace().");
|
|
39320
|
+
}
|
|
39321
|
+
const loaded = await loadLoCoMoDataset("full", options.datasetDir);
|
|
39322
|
+
const multiHopRecallComposition = options.multiHopRecallComposition ?? true;
|
|
39323
|
+
const plans = loaded.items.map((conversation) => buildLoCoMoPlan(conversation, multiHopRecallComposition));
|
|
39324
|
+
const selectable = plans.flatMap(
|
|
39325
|
+
(plan, planIndex) => plan.trials.map(
|
|
39326
|
+
(trial) => ({
|
|
39327
|
+
taskId: trial.taskId,
|
|
39328
|
+
question: trial.question,
|
|
39329
|
+
recallSessionIds: [...trial.recallSessionIds],
|
|
39330
|
+
planIndex
|
|
39331
|
+
})
|
|
39332
|
+
)
|
|
39333
|
+
);
|
|
39334
|
+
const selection = selectLoCoMoRetrievalTraceTasks(selectable, options.selector);
|
|
39335
|
+
const selectedIds = new Set(selection.selectedTaskIds);
|
|
39336
|
+
const tasks = [];
|
|
39337
|
+
for (let planIndex = 0; planIndex < plans.length; planIndex += 1) {
|
|
39338
|
+
const selected = selectable.filter((task) => task.planIndex === planIndex && selectedIds.has(task.taskId));
|
|
39339
|
+
if (selected.length === 0) continue;
|
|
39340
|
+
const plan = plans[planIndex];
|
|
39341
|
+
if (!plan) throw new Error(`Missing LoCoMo plan at index ${planIndex}.`);
|
|
39342
|
+
await options.system.reset();
|
|
39343
|
+
for (const session of plan.ingestSessions) {
|
|
39344
|
+
if (session.messages.length > 0) {
|
|
39345
|
+
await options.system.store(session.sessionId, session.messages);
|
|
39346
|
+
}
|
|
39347
|
+
}
|
|
39348
|
+
await options.system.drain?.();
|
|
39349
|
+
for (const selectedTask of selected) {
|
|
39350
|
+
const recallBudgetChars = benchmarkRecallBudgetForSessionCount(selectedTask.recallSessionIds.length);
|
|
39351
|
+
const recalled = await Promise.all(
|
|
39352
|
+
selectedTask.recallSessionIds.map(async (sessionId) => {
|
|
39353
|
+
const result = await recallWithTrace(sessionId, selectedTask.question, recallBudgetChars);
|
|
39354
|
+
return {
|
|
39355
|
+
text: result.text,
|
|
39356
|
+
receipt: {
|
|
39357
|
+
session: digestContent(sessionId),
|
|
39358
|
+
trace: sanitizeStructuralTrace(result.trace)
|
|
39359
|
+
}
|
|
39360
|
+
};
|
|
39361
|
+
})
|
|
39362
|
+
);
|
|
39363
|
+
const rawRecalledText = recalled.map((entry) => entry.text).filter(Boolean).join("\n\n");
|
|
39364
|
+
const sanitized = sanitizeLoCoMoRecallText({
|
|
39365
|
+
question: selectedTask.question,
|
|
39366
|
+
recalledText: rawRecalledText
|
|
39367
|
+
});
|
|
39368
|
+
const composition = prioritizeLoCoMoRecallTextWithTrace({
|
|
39369
|
+
question: selectedTask.question,
|
|
39370
|
+
recalledText: sanitized,
|
|
39371
|
+
multiHopRecallComposition
|
|
39372
|
+
});
|
|
39373
|
+
tasks.push({
|
|
39374
|
+
taskId: selectedTask.taskId,
|
|
39375
|
+
question: digestContent(selectedTask.question),
|
|
39376
|
+
recallBudgetChars,
|
|
39377
|
+
sessions: recalled.map((entry) => entry.receipt),
|
|
39378
|
+
composition: composition.receipt
|
|
39379
|
+
});
|
|
39380
|
+
}
|
|
39381
|
+
}
|
|
39382
|
+
const withoutHash = {
|
|
39383
|
+
schemaVersion: LOCOMO_RETRIEVAL_TRACE_SCHEMA_VERSION,
|
|
39384
|
+
benchmarkId: "locomo",
|
|
39385
|
+
captureKind: "retrieval-only",
|
|
39386
|
+
sensitivity: {
|
|
39387
|
+
classification: "restricted",
|
|
39388
|
+
contentEncoding: "sha256+length",
|
|
39389
|
+
containsGold: false,
|
|
39390
|
+
containsRawContent: false
|
|
39391
|
+
},
|
|
39392
|
+
provenance: {
|
|
39393
|
+
gitSha: options.gitSha,
|
|
39394
|
+
remnicVersion: options.remnicVersion,
|
|
39395
|
+
runtimeProfile: options.runtimeProfile,
|
|
39396
|
+
adapterMode: "direct",
|
|
39397
|
+
replayExtractionMode: "skip",
|
|
39398
|
+
providerFree: true,
|
|
39399
|
+
dataset: { id: "locomo-10", sha256: loaded.sha256 },
|
|
39400
|
+
retrievalConfigSha256: hashCanonicalJson(retrievalConfig),
|
|
39401
|
+
recallBudget: {
|
|
39402
|
+
algorithm: "benchmarkRecallBudgetForSessionCount",
|
|
39403
|
+
version: LOCOMO_RETRIEVAL_TRACE_BUDGET_VERSION
|
|
39404
|
+
}
|
|
39405
|
+
},
|
|
39406
|
+
selection,
|
|
39407
|
+
tasks
|
|
39408
|
+
};
|
|
39409
|
+
return {
|
|
39410
|
+
...withoutHash,
|
|
39411
|
+
artifactHash: hashCanonicalJson(withoutHash)
|
|
39412
|
+
};
|
|
39413
|
+
}
|
|
39414
|
+
function selectLoCoMoRetrievalTraceTasks(tasks, selector) {
|
|
39415
|
+
const allIds = tasks.map((task) => task.taskId);
|
|
39416
|
+
if (new Set(allIds).size !== allIds.length) {
|
|
39417
|
+
throw new Error("LoCoMo retrieval trace task ids must be unique.");
|
|
39418
|
+
}
|
|
39419
|
+
let selected;
|
|
39420
|
+
let algorithm;
|
|
39421
|
+
let seed;
|
|
39422
|
+
const hasTaskIds = "taskIds" in selector && selector.taskIds !== void 0;
|
|
39423
|
+
const hasSampleSize = "sampleSize" in selector && selector.sampleSize !== void 0;
|
|
39424
|
+
if (Number(hasTaskIds) + Number(hasSampleSize) !== 1) {
|
|
39425
|
+
throw new Error("Choose exactly one LoCoMo retrieval trace selector.");
|
|
39426
|
+
}
|
|
39427
|
+
if (hasTaskIds && "seed" in selector && selector.seed !== void 0) {
|
|
39428
|
+
throw new Error("LoCoMo retrieval trace seed is valid only for seeded sampling.");
|
|
39429
|
+
}
|
|
39430
|
+
if (hasTaskIds) {
|
|
39431
|
+
algorithm = "explicit-task-ids";
|
|
39432
|
+
const requestedTaskIds = selector.taskIds;
|
|
39433
|
+
if (!requestedTaskIds) throw new Error("LoCoMo explicit task ids are required.");
|
|
39434
|
+
const requested = [...requestedTaskIds];
|
|
39435
|
+
if (requested.length === 0) {
|
|
39436
|
+
throw new Error("LoCoMo retrieval trace explicit task selection cannot be empty.");
|
|
39437
|
+
}
|
|
39438
|
+
if (new Set(requested).size !== requested.length) {
|
|
39439
|
+
throw new Error("LoCoMo retrieval trace explicit task ids must not contain duplicates.");
|
|
39440
|
+
}
|
|
39441
|
+
const available = new Set(allIds);
|
|
39442
|
+
const unknown = requested.filter((taskId) => !available.has(taskId));
|
|
39443
|
+
if (unknown.length > 0) {
|
|
39444
|
+
throw new Error(`Unknown LoCoMo retrieval trace task id: ${unknown[0]}`);
|
|
39445
|
+
}
|
|
39446
|
+
const requestedSet = new Set(requested);
|
|
39447
|
+
selected = allIds.filter((taskId) => requestedSet.has(taskId));
|
|
39448
|
+
} else {
|
|
39449
|
+
algorithm = "sha256-seeded-sample";
|
|
39450
|
+
const sampleSize = selector.sampleSize;
|
|
39451
|
+
seed = selector.seed;
|
|
39452
|
+
if (sampleSize === void 0 || seed === void 0) {
|
|
39453
|
+
throw new Error("LoCoMo seeded sampling requires sampleSize and seed.");
|
|
39454
|
+
}
|
|
39455
|
+
if (!Number.isSafeInteger(sampleSize) || sampleSize <= 0 || sampleSize > allIds.length) {
|
|
39456
|
+
throw new Error(`LoCoMo retrieval trace sampleSize must be an integer from 1 to ${allIds.length}.`);
|
|
39457
|
+
}
|
|
39458
|
+
if (!Number.isSafeInteger(seed) || seed < 0) {
|
|
39459
|
+
throw new Error("LoCoMo retrieval trace seed must be a non-negative safe integer.");
|
|
39460
|
+
}
|
|
39461
|
+
const sampled = [...allIds].sort((left, right) => {
|
|
39462
|
+
const leftHash = hashString(`${seed}\0${left}`);
|
|
39463
|
+
const rightHash = hashString(`${seed}\0${right}`);
|
|
39464
|
+
return leftHash.localeCompare(rightHash) || left.localeCompare(right);
|
|
39465
|
+
}).slice(0, sampleSize);
|
|
39466
|
+
const sampledSet = new Set(sampled);
|
|
39467
|
+
selected = allIds.filter((taskId) => sampledSet.has(taskId));
|
|
39468
|
+
}
|
|
39469
|
+
return {
|
|
39470
|
+
algorithm,
|
|
39471
|
+
version: LOCOMO_RETRIEVAL_TRACE_SELECTION_VERSION,
|
|
39472
|
+
...seed === void 0 ? {} : { seed },
|
|
39473
|
+
candidateCount: allIds.length,
|
|
39474
|
+
selectedCount: selected.length,
|
|
39475
|
+
selectedTaskIds: selected,
|
|
39476
|
+
selectedTaskIdsSha256: hashCanonicalJson(selected)
|
|
39477
|
+
};
|
|
39478
|
+
}
|
|
39479
|
+
function serializeLoCoMoRetrievalTraceReceipt(receipt) {
|
|
39480
|
+
return `${canonicalJsonStringify(receipt, 2)}
|
|
39481
|
+
`;
|
|
39482
|
+
}
|
|
39483
|
+
function sanitizeStructuralTrace(trace2) {
|
|
39484
|
+
return {
|
|
39485
|
+
schemaVersion: trace2.schemaVersion,
|
|
39486
|
+
sensitivity: { ...trace2.sensitivity },
|
|
39487
|
+
sections: trace2.sections.map((section) => ({ ...section })),
|
|
39488
|
+
selections: trace2.selections.map(({ summary, ...selection }) => ({
|
|
39489
|
+
...selection,
|
|
39490
|
+
...selection.archiveRowIds === void 0 ? {} : { archiveRowIds: [...selection.archiveRowIds] },
|
|
39491
|
+
...summary === void 0 ? {} : { summary: { depth: summary.depth, msgStart: summary.msgStart, msgEnd: summary.msgEnd } }
|
|
39492
|
+
})),
|
|
39493
|
+
lcmCandidates: trace2.lcmCandidates.map((candidate) => ({ ...candidate })),
|
|
39494
|
+
...trace2.coreCapture === void 0 ? {} : {
|
|
39495
|
+
coreCapture: {
|
|
39496
|
+
budget: { ...trace2.coreCapture.budget },
|
|
39497
|
+
filters: trace2.coreCapture.filters.map((filter) => ({ ...filter })),
|
|
39498
|
+
results: trace2.coreCapture.results.map((result) => {
|
|
39499
|
+
assertMemoryIdRef(result.memoryIdRef);
|
|
39500
|
+
const score = result.scoreDecomposition;
|
|
39501
|
+
return {
|
|
39502
|
+
memoryIdRef: {
|
|
39503
|
+
sha256: result.memoryIdRef.sha256,
|
|
39504
|
+
length: result.memoryIdRef.length
|
|
39505
|
+
},
|
|
39506
|
+
servedBy: result.servedBy,
|
|
39507
|
+
scoreDecomposition: {
|
|
39508
|
+
...score.vector === void 0 ? {} : { vector: score.vector },
|
|
39509
|
+
...score.bm25 === void 0 ? {} : { bm25: score.bm25 },
|
|
39510
|
+
...score.importance === void 0 ? {} : { importance: score.importance },
|
|
39511
|
+
...score.mmrPenalty === void 0 ? {} : { mmrPenalty: score.mmrPenalty },
|
|
39512
|
+
...score.tierPrior === void 0 ? {} : { tierPrior: score.tierPrior },
|
|
39513
|
+
...score.reinforcementBoost === void 0 ? {} : { reinforcementBoost: score.reinforcementBoost },
|
|
39514
|
+
final: score.final
|
|
39515
|
+
},
|
|
39516
|
+
admittedBy: [...result.admittedBy],
|
|
39517
|
+
...result.rejectedBy === void 0 ? {} : { rejectedBy: result.rejectedBy },
|
|
39518
|
+
...result.disclosure === void 0 ? {} : { disclosure: result.disclosure },
|
|
39519
|
+
...result.estimatedTokens === void 0 ? {} : { estimatedTokens: result.estimatedTokens }
|
|
39520
|
+
};
|
|
39521
|
+
})
|
|
39522
|
+
}
|
|
39523
|
+
},
|
|
39524
|
+
budget: { ...trace2.budget }
|
|
39525
|
+
};
|
|
39526
|
+
}
|
|
39527
|
+
function digestContent(value) {
|
|
39528
|
+
return {
|
|
39529
|
+
sha256: hashString(value),
|
|
39530
|
+
charCount: value.length,
|
|
39531
|
+
lineCount: value.length === 0 ? 0 : value.split("\n").length
|
|
39532
|
+
};
|
|
39533
|
+
}
|
|
39534
|
+
function assertCaptureOptions(options) {
|
|
39535
|
+
if (!options.datasetDir.trim()) {
|
|
39536
|
+
throw new Error("LoCoMo retrieval trace capture requires datasetDir.");
|
|
39537
|
+
}
|
|
39538
|
+
if (options.runtimeProfile !== "baseline" && options.runtimeProfile !== "real") {
|
|
39539
|
+
throw new Error('LoCoMo retrieval trace runtimeProfile must be "baseline" or "real".');
|
|
39540
|
+
}
|
|
39541
|
+
if (!options.gitSha.trim() || !options.remnicVersion.trim() || options.gitSha === "unknown" || options.remnicVersion === "unknown") {
|
|
39542
|
+
throw new Error("LoCoMo retrieval trace provenance requires gitSha and remnicVersion.");
|
|
39543
|
+
}
|
|
39544
|
+
if (options.providerFreeConfirmed !== true) {
|
|
39545
|
+
throw new Error("LoCoMo retrieval trace capture requires explicit provider-free confirmation.");
|
|
39546
|
+
}
|
|
39547
|
+
}
|
|
39548
|
+
function assertProviderFreeRetrievalConfig(value) {
|
|
39549
|
+
const config = assertJsonConfig(value);
|
|
39550
|
+
for (const key of [
|
|
39551
|
+
"localLlmEnabled",
|
|
39552
|
+
"localLlmFastEnabled",
|
|
39553
|
+
"recallPlannerEnabled",
|
|
39554
|
+
"embeddingFallbackEnabled",
|
|
39555
|
+
"hostEmbeddingProviderEnabled",
|
|
39556
|
+
"openaiApiKey"
|
|
39557
|
+
]) {
|
|
39558
|
+
if (config[key] !== false) {
|
|
39559
|
+
throw new Error(`retrievalConfig.${key} must be false for provider-free capture.`);
|
|
39560
|
+
}
|
|
39561
|
+
}
|
|
39562
|
+
if (config.modelSource !== "plugin") {
|
|
39563
|
+
throw new Error('retrievalConfig.modelSource must be "plugin" for provider-free capture.');
|
|
39564
|
+
}
|
|
39565
|
+
return config;
|
|
39566
|
+
}
|
|
39567
|
+
function assertMemoryIdRef(value) {
|
|
39568
|
+
if (!value || typeof value !== "object" || !/^[0-9a-f]{64}$/u.test(value.sha256) || !Number.isSafeInteger(value.length) || value.length <= 0) {
|
|
39569
|
+
throw new Error("LoCoMo retrieval trace requires a valid content-free memoryIdRef.");
|
|
39570
|
+
}
|
|
39571
|
+
}
|
|
39572
|
+
function assertJsonConfig(value, path40 = "retrievalConfig") {
|
|
39573
|
+
if (value === null || typeof value === "string" || typeof value === "boolean") return value;
|
|
39574
|
+
if (typeof value === "number") {
|
|
39575
|
+
if (!Number.isFinite(value)) throw new Error(`${path40} must contain only finite JSON numbers.`);
|
|
39576
|
+
return value;
|
|
39577
|
+
}
|
|
39578
|
+
if (Array.isArray(value)) {
|
|
39579
|
+
return value.map((entry, index) => assertJsonConfig(entry, `${path40}[${index}]`));
|
|
39580
|
+
}
|
|
39581
|
+
if (!value || typeof value !== "object") {
|
|
39582
|
+
throw new Error(`${path40} must be JSON-serializable and provider-free.`);
|
|
39583
|
+
}
|
|
39584
|
+
const output = {};
|
|
39585
|
+
for (const key of Object.keys(value).sort()) {
|
|
39586
|
+
const child = value[key];
|
|
39587
|
+
if (key === "openaiApiKey") {
|
|
39588
|
+
if (child !== false) {
|
|
39589
|
+
throw new Error(`${path40}.${key} must be exactly false for provider-free capture.`);
|
|
39590
|
+
}
|
|
39591
|
+
output[key] = false;
|
|
39592
|
+
continue;
|
|
39593
|
+
}
|
|
39594
|
+
if (isSecretKey(key)) {
|
|
39595
|
+
throw new Error(`${path40}.${key} contains secret-bearing configuration.`);
|
|
39596
|
+
}
|
|
39597
|
+
if (child === void 0) continue;
|
|
39598
|
+
if (/^(?:gatewayConfig|gatewayAgentId|fastGatewayAgentId|internalProvider|llmProvider|llmModel)$/iu.test(key) || key === "modelSource" && child !== "plugin") {
|
|
39599
|
+
throw new Error(`${path40}.${key} is provider-capable configuration.`);
|
|
39600
|
+
}
|
|
39601
|
+
output[key] = assertJsonConfig(child, `${path40}.${key}`);
|
|
39602
|
+
}
|
|
39603
|
+
return output;
|
|
39604
|
+
}
|
|
39605
|
+
function sanitizeProviderFreeRetrievalConfig(value, path40 = "retrievalConfig") {
|
|
39606
|
+
if (value === null || typeof value === "string" || typeof value === "boolean") return value;
|
|
39607
|
+
if (typeof value === "number") {
|
|
39608
|
+
if (!Number.isFinite(value)) throw new Error(`${path40} must contain only finite JSON numbers.`);
|
|
39609
|
+
return value;
|
|
39610
|
+
}
|
|
39611
|
+
if (Array.isArray(value)) {
|
|
39612
|
+
return value.map((entry, index) => sanitizeProviderFreeRetrievalConfig(entry, `${path40}[${index}]`));
|
|
39613
|
+
}
|
|
39614
|
+
if (!value || typeof value !== "object") {
|
|
39615
|
+
throw new Error(`${path40} must be JSON-serializable.`);
|
|
39616
|
+
}
|
|
39617
|
+
const output = {};
|
|
39618
|
+
for (const key of Object.keys(value).sort()) {
|
|
39619
|
+
const child = value[key];
|
|
39620
|
+
if (child === void 0) continue;
|
|
39621
|
+
if (isSecretKey(key) || /^(?:gatewayConfig|gatewayAgentId|fastGatewayAgentId|internalProvider|llmProvider|llmModel)$/iu.test(key) || key === "modelSource") {
|
|
39622
|
+
continue;
|
|
39623
|
+
}
|
|
39624
|
+
output[key] = sanitizeProviderFreeRetrievalConfig(child, `${path40}.${key}`);
|
|
39625
|
+
}
|
|
39626
|
+
return output;
|
|
39627
|
+
}
|
|
39628
|
+
|
|
39197
39629
|
// src/integrity/sealed-qrels.ts
|
|
39198
39630
|
import { readFile as readFile20 } from "fs/promises";
|
|
39199
39631
|
function isSealedQrelsArtifact(value) {
|
|
@@ -43064,6 +43496,9 @@ export {
|
|
|
43064
43496
|
LOCOMO_FULL_TASK_COUNT,
|
|
43065
43497
|
LOCOMO_RECALL_DIFF_LINE_LIMIT,
|
|
43066
43498
|
LOCOMO_RECALL_EXCERPT_CHARS,
|
|
43499
|
+
LOCOMO_RETRIEVAL_TRACE_BUDGET_VERSION,
|
|
43500
|
+
LOCOMO_RETRIEVAL_TRACE_SCHEMA_VERSION,
|
|
43501
|
+
LOCOMO_RETRIEVAL_TRACE_SELECTION_VERSION,
|
|
43067
43502
|
LONG_MEM_EVAL_DATASET_FILENAMES,
|
|
43068
43503
|
LettaMemCorrectAdapter,
|
|
43069
43504
|
LocalLabPreflightError,
|
|
@@ -43120,10 +43555,12 @@ export {
|
|
|
43120
43555
|
buildCodexCreditReceipt,
|
|
43121
43556
|
buildJudgePayload,
|
|
43122
43557
|
buildOracleTrajectoryRecall,
|
|
43558
|
+
buildProviderFreeLoCoMoRetrievalConfig,
|
|
43123
43559
|
buildSchemaTierFixture,
|
|
43124
43560
|
buildSchemaTierSmokeFixture,
|
|
43125
43561
|
calendarFixture,
|
|
43126
43562
|
canonicalJsonStringify,
|
|
43563
|
+
captureLoCoMoRetrievalTrace,
|
|
43127
43564
|
captureMachineFingerprint,
|
|
43128
43565
|
chatFixture,
|
|
43129
43566
|
checkCodingGraphRegression,
|
|
@@ -43192,6 +43629,7 @@ export {
|
|
|
43192
43629
|
getAblationCell,
|
|
43193
43630
|
getBenchmark,
|
|
43194
43631
|
getBenchmarkLowerIsBetter,
|
|
43632
|
+
getGitSha,
|
|
43195
43633
|
getMemoryEvalDimension,
|
|
43196
43634
|
getProviderBackedJudgePromptIdentity,
|
|
43197
43635
|
getRemnicVersion,
|
|
@@ -43245,6 +43683,7 @@ export {
|
|
|
43245
43683
|
parseSealedQrels,
|
|
43246
43684
|
pickStableQualifiedName,
|
|
43247
43685
|
precisionAtK,
|
|
43686
|
+
preflightLoCoMoRetrievalTraceCapture,
|
|
43248
43687
|
preflightLocalLabRole,
|
|
43249
43688
|
projectFolderFixture,
|
|
43250
43689
|
recallAtK,
|
|
@@ -43300,6 +43739,7 @@ export {
|
|
|
43300
43739
|
selectFixtureVariant,
|
|
43301
43740
|
serializeBenchmarkArtifact,
|
|
43302
43741
|
serializeJsonl,
|
|
43742
|
+
serializeLoCoMoRetrievalTraceReceipt,
|
|
43303
43743
|
serializeSealedQrels,
|
|
43304
43744
|
shuffleTasks,
|
|
43305
43745
|
timed,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@remnic/bench",
|
|
3
|
-
"version": "9.6.
|
|
3
|
+
"version": "9.6.33",
|
|
4
4
|
"description": "Retrieval latency ladder benchmarks + CI regression gates for @remnic/core",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -39,8 +39,8 @@
|
|
|
39
39
|
"hyparquet": "^1.25.7",
|
|
40
40
|
"yaml": "^2.4.2",
|
|
41
41
|
"zod": "^3.24.0",
|
|
42
|
-
"@remnic/coding-graph": "^9.6.
|
|
43
|
-
"@remnic/core": "^9.6.
|
|
42
|
+
"@remnic/coding-graph": "^9.6.33",
|
|
43
|
+
"@remnic/core": "^9.6.33"
|
|
44
44
|
},
|
|
45
45
|
"devDependencies": {
|
|
46
46
|
"tsup": "^8.5.1",
|