@remnic/bench 9.3.681 → 9.3.682

package/dist/index.d.ts

@@ -211,7 +211,7 @@ type BenchmarkMode = "full" | "quick";
 type BenchmarkTier = "published" | "remnic" | "custom";
 type BenchmarkStatus = "ready" | "planned";
 type BenchmarkCategory = "agentic" | "retrieval" | "conversational" | "ingestion";
-type BenchRuntimeProfile = "baseline" | "real" | "openclaw-chain";
+type BenchRuntimeProfile = "baseline" | "real" | "openclaw-chain" | "local-lab";
 type AmaBenchJudgeProtocol = "default" | "recommended";
 /**
  * Built-in LLM providers supported by the bench harness.
@@ -244,6 +244,18 @@ interface ProviderConfig {
     reasoningEffort?: BenchReasoningEffort;
     responderContextBudgetChars?: number;
     responderPromptBudgetChars?: number;
+    /**
+     * Sampling temperature forwarded from a runtime profile manifest (e.g. the
+     * local-lab manifest pins this to 0 for reproducibility). Optional; providers
+     * that do not read it ignore the value.
+     */
+    temperature?: number;
+    /**
+     * Sampling seed forwarded from a runtime profile manifest so local-lab runs
+     * are reproducible across invocations. Optional; providers that do not read
+     * it ignore the value.
+     */
+    seed?: number;
 }
 interface TaskTokenUsage {
     input: number;
@@ -648,6 +660,18 @@ interface ProviderBaseConfig {
     };
     /** Suppress thinking/reasoning tokens for thinking-capable models (Qwen 3.5, Gemma 4, DeepSeek). */
     disableThinking?: boolean;
+    /**
+     * Sampling temperature pinned by a runtime profile manifest (e.g. local-lab
+     * pins 0 for reproducibility). Providers use this as the fallback when the
+     * per-call CompletionOpts does not override it.
+     */
+    temperature?: number;
+    /**
+     * Sampling seed pinned by a runtime profile manifest so reruns reproduce the
+     * same draws. Providers forward this to the backend on every completion call
+     * (Ollama options.seed, OpenAI-compatible top-level seed).
+     */
+    seed?: number;
     /**
      * Optional answering-only memory-context budget. Benchmark artifacts keep the
      * full recalled text, but provider-backed responders may receive this compact
@@ -1723,6 +1747,340 @@ interface AssistantRunnerOptions {
     random?: () => number;
 }
 
+/**
+ * Local-lab runtime profile manifest (issue #1573 PR2).
+ *
+ * A `local-lab` profile is a JSON manifest — never hardcoded model strings
+ * (rule 30/55) — that pins a single-locale bench run (responder, judge,
+ * optional embedding) to operator-hosted models. It is resolved by
+ * `resolveBenchRuntimeProfile` to drive sequential phase scheduling and
+ * endpoint preflight (see `preflight.ts` and `sequential-phases.ts`).
+ *
+ * Field contract:
+ *
+ *   - `provider` is one of `LOCAL_LAB_PROVIDER_KINDS` ("openai-compatible"
+ *     for llama.cpp / vLLM / LM Studio; "ollama" for native Ollama). Any
+ *     other value is REJECTED with the valid kinds listed (rule 51 — never
+ *     silently fall through to a default).
+ *   - `temperature` is pinned to 0 and `seed` is required so local-lab
+ *     runs are reproducible.
+ *   - `ctx` is the manifest-declared serving context (tokens). Preflight
+ *     verifies the live endpoint reports at least this much.
+ *
+ * The manifest is content (no command strings interpolated into shells;
+ * rule 10) — `baseUrl`/`model` are only ever fetch targets.
+ */
+/**
+ * Provider kinds accepted by a local-lab manifest role.
+ *
+ * `"openai-compatible"` maps to the OpenAI-compatible transport
+ * (`/v1/chat/completions` + `/v1/models`) used by llama.cpp, vLLM, LM
+ * Studio, etc. `"ollama"` maps to Ollama's native transport
+ * (`/api/generate` + `/api/tags`).
+ */
+declare const LOCAL_LAB_PROVIDER_KINDS: readonly ["openai-compatible", "ollama"];
+type LocalLabProviderKind = (typeof LOCAL_LAB_PROVIDER_KINDS)[number];
+interface LocalLabRoleConfig {
+    provider: LocalLabProviderKind;
+    /** Base URL of the operator-hosted endpoint (e.g. `http://localhost:1234/v1`). */
+    baseUrl: string;
+    /** Exact model id the endpoint reports (no aliases, no shell interpolation). */
+    model: string;
+    /** Optional quantization label (informational; recorded in artifacts). */
+    quantization?: string;
+    /** Manifest-declared serving context length in tokens. */
+    ctx: number;
+    /** Sampling temperature. Local-lab pins this to 0 for reproducibility. */
+    temperature: 0;
+    /** Sampling seed; required so reruns reproduce the same draws. */
+    seed: number;
+}
+interface LocalLabManifestNotes {
+    /**
+     * Free-form operator guidance printed between responder and judge phases
+     * when the two roles live on different endpoints. When both roles share
+     * an endpoint the runner skips the hand-off (see sequential-phases.ts).
+     */
+    responderToJudgeHandoff?: string;
+    [key: string]: unknown;
+}
+interface LocalLabManifest {
+    /** Manifest discriminator; always the literal `"local-lab"`. */
+    profile: "local-lab";
+    responder: LocalLabRoleConfig;
+    judge: LocalLabRoleConfig;
+    embedding?: LocalLabRoleConfig;
+    /** Phase scheduling mode. PR2 ships `"sequential"` only. */
+    phases: "sequential";
+    notes?: LocalLabManifestNotes;
+}
+/**
+ * Parse and validate a local-lab manifest from an unknown parsed JSON value.
+ * Throws a rule-51-shaped error (lists valid kinds) on any violation.
+ */
+declare function parseLocalLabManifest(raw: unknown): LocalLabManifest;
+/**
+ * Read and parse a local-lab manifest from disk. The path is opened read-only;
+ * nothing in the manifest is ever interpolated into a shell (rule 10).
+ */
+declare function loadLocalLabManifest(filePath: string): Promise<LocalLabManifest>;
+
+/**
+ * Resolve a parsed local-lab manifest into runtime ProviderConfigs.
+ *
+ * Resolution is the bridge between the manifest (operator-authored JSON) and
+ * the harness's existing `ProviderConfig` shape: each manifest role becomes a
+ * `ProviderConfig` with `temperature` and `seed` forwarded verbatim so a test
+ * can assert on the resolved config (issue #1573 PR2 test list).
+ *
+ * Provider kind mapping (kept dumb and explicit, never silent fallback):
+ *
+ *   - `"openai-compatible"` → `BuiltInProvider` `"local-llm"` (the bench
+ *     provider that talks `/v1/chat/completions` + `/v1/models`, requiring an
+ *     explicit `baseUrl` — exactly what the manifest pins).
+ *   - `"ollama"` → `"ollama"` (native `/api/generate` + `/api/tags`).
+ *
+ * Both baseUrl and model are copied as-is; they are only ever fetch targets,
+ * never interpolated into a shell (rule 10). API keys are not part of the
+ * manifest — local-lab endpoints are operator-hosted on the loopback or a
+ * private host, so persisting a key into the manifest would be a footgun.
+ * Operators with auth'd local endpoints pass the key out-of-band.
+ *
+ * `quantization` is informational only; the bench `ProviderConfig` does not
+ * have a quantization field, so it is kept on the resolved role and surfaces
+ * in the bench artifact (PR3's tier/hardware metadata).
+ */
+
+/**
+ * A manifest role paired with its resolved `ProviderConfig`. The PR2 test
+ * list asserts temperature/seed on `providerConfig` directly.
+ */
+interface ResolvedLocalLabRole {
+    readonly provider: LocalLabRoleConfig["provider"];
+    readonly baseUrl: string;
+    readonly model: string;
+    readonly quantization?: string;
+    readonly ctx: number;
+    readonly temperature: number;
+    readonly seed: number;
+    readonly providerConfig: ProviderConfig;
+}
+interface ResolvedLocalLabProfile {
+    /** The parsed manifest this resolution was produced from. */
+    readonly manifest: LocalLabManifest;
+    readonly responder: ResolvedLocalLabRole;
+    readonly judge: ResolvedLocalLabRole;
+    readonly embedding?: ResolvedLocalLabRole;
+    /** Phase scheduling mode. PR2 ships `"sequential"` only. */
+    readonly phases: "sequential";
+    /** Operator hand-off note (or undefined when not authored). */
+    readonly notes?: LocalLabManifestNotes;
+}
+/**
+ * Resolve a single manifest role into a `ResolvedLocalLabRole`, forwarding
+ * `temperature` and `seed` into the `ProviderConfig` so providers can read
+ * them off the config directly.
+ *
+ * Ollama `baseUrl`s are normalized to include `/api` so the provider posts to
+ * `${baseUrl}/generate` → `…/api/generate` rather than `…/generate` (404).
+ * This mirrors `discoveryEndpointFor` which already appends `/api/tags` for
+ * preflight. Operators can write `http://127.0.0.1:11434` or
+ * `http://127.0.0.1:11434/api` interchangeably (codex review, #1573 PR2).
+ */
+declare function resolveLocalLabRole(role: LocalLabRoleConfig): ResolvedLocalLabRole;
+/**
+ * Resolve the full manifest into a `ResolvedLocalLabProfile`. Used by
+ * `resolveBenchRuntimeProfile` for `runtimeProfile: "local-lab"`, and
+ * directly by the unit test for the temperature/seed forwarding assertion.
+ */
+declare function resolveLocalLabProfile(manifest: LocalLabManifest): ResolvedLocalLabProfile;
+
+/**
+ * Local-lab endpoint preflight (issue #1573 PR2).
+ *
+ * Before each phase the runner verifies the operator-hosted endpoint is
+ * actually serving the model the manifest claims, with at least the
+ * manifest-declared context length. The harness never manages model
+ * processes — it asks the endpoint what's live and refuses to proceed
+ * ("hard error listing what was found vs expected", rule 51) on any
+ * mismatch. Silent fallback is explicitly forbidden.
+ *
+ * Discovery endpoints by provider kind:
+ *
+ *   - `"openai-compatible"` → `GET <baseUrl>/models` (or `/v1/models` when
+ *     the baseUrl does not already end in `/v1`). The body shape mirrors the
+ *     OpenAI `/v1/models` contract: `{ data: [{ id, context_length?, ... }] }`.
+ *   - `"ollama"` → `GET <baseUrl>/tags` (or `/api/tags`). The body shape
+ *     mirrors Ollama's `/api/tags` contract: `{ models: [{ name,
+ *     details?.parameter_size?, ... }] }`.
+ *
+ * `baseUrl` is composed into a fetch URL only — never a shell string
+ * (rule 10). Failures carry the endpoint's actual reported model list so an
+ * operator can immediately see why their manifest doesn't match.
+ */
+
+/** Minimal shape the preflight reader needs from a discovered model entry. */
+interface PreflightDiscoveredModel {
+    id: string;
+    contextLength?: number;
+}
+interface LocalLabPreflightInput {
+    provider: LocalLabProviderKind;
+    baseUrl: string;
+    /** Exact model id the endpoint is expected to report. */
+    model: string;
+    /** Manifest-declared serving context length; the live endpoint must meet or exceed it. */
+    ctx: number;
+}
+interface LocalLabPreflightSuccess {
+    ok: true;
+    provider: LocalLabProviderKind;
+    endpoint: string;
+    expectedModel: string;
+    foundModels: PreflightDiscoveredModel[];
+    /** Resolved context length for the matched model, when the endpoint reports one. */
+    matchedContextLength?: number;
+}
+interface LocalLabPreflightFailure {
+    ok: false;
+    provider: LocalLabProviderKind;
+    endpoint: string;
+    expectedModel: string;
+    foundModels: PreflightDiscoveredModel[];
+    expectedCtx: number;
+    matchedContextLength?: number;
+    reason: string;
+}
+type LocalLabPreflightResult = LocalLabPreflightSuccess | LocalLabPreflightFailure;
+interface LocalLabPreflightOptions {
+    signal?: AbortSignal;
+    /** Per-request timeout. Defaults to 5 s — preflight should be fast. */
+    timeoutMs?: number;
+    /** Inject a fetch implementation (tests). Defaults to global fetch. */
+    fetchImpl?: typeof fetch;
+}
+/**
+ * Preflight a single manifest role. Resolves to a result object — never
+ * throws for endpoint/discovery failures (those are preflight failures, not
+ * runtime errors). Throws only on truly exceptional conditions (invalid
+ * baseUrl shape, fetchImpl contract violation).
+ */
+declare function preflightLocalLabRole(input: LocalLabPreflightInput, options?: LocalLabPreflightOptions): Promise<LocalLabPreflightResult>;
+/**
+ * Compose the discovery URL for a provider kind + baseUrl. Never
+ * interpolates into a shell — only into a fetch URL. Tolerant of trailing
+ * slashes and the common `/v1` (OpenAI-compatible) / `/api` (Ollama)
+ * suffixes already being present.
+ */
+declare function discoveryEndpointFor(provider: LocalLabProviderKind, baseUrl: string): string;
+
+/**
+ * Local-lab sequential phase scheduler (issue #1573 PR2).
+ *
+ * On a single-GPU lab box (e.g. RTX 3090, 24 GB VRAM) the responder and
+ * judge models cannot be co-resident: the harness runs them in two distinct
+ * phases, with the operator physically swapping which model is loaded
+ * between them. The harness DOES NOT manage model processes — it tells the
+ * operator what to do and waits for them to confirm via the next phase's
+ * endpoint preflight succeeding.
+ *
+ * The scheduler:
+ *
+ *   1. Preflights the phase's endpoint (reachable + serving the manifest
+ *      model with enough context — see `preflight.ts`). Hard error on
+ *      mismatch; no silent fallback.
+ *   2. Calls the phase's `execute()` callback — the runner supplies the
+ *      actual ingest/answer or judge work; the scheduler only sequences.
+ *   3. Between phases prints the operator hand-off note from
+ *      `manifest.notes.responderToJudgeHandoff`, OR proceeds silently when
+ *      the next phase's endpoint is the same baseUrl (Ollama can hot-swap
+ *      models within one endpoint, so no physical swap is needed).
+ *
+ * `execute` receives the resolved role so the runner can build the right
+ * provider config without re-reading the manifest. The phase scheduler is
+ * intentionally process-supervision-free per the issue's "do not add
+ * process supervision in v1" instruction.
+ */
+
+type LocalLabPhaseName = "responder" | "judge" | "embedding";
+interface LocalLabPhaseDescriptor {
+    name: LocalLabPhaseName;
+    role: LocalLabRoleConfig;
+}
+/**
+ * A unit of phase work supplied by the runner. The callback receives the
+ * phase's resolved role so the caller can build its own provider/client
+ * without re-reading the manifest. Returning a value (e.g. judge verdicts)
+ * is supported; the scheduler does not inspect it.
+ */
+type LocalLabPhaseExecute<T = unknown> = (role: ResolvedLocalLabRole) => Promise<T>;
+interface LocalLabPhase<T = unknown> {
+    name: LocalLabPhaseName;
+    role: LocalLabRoleConfig;
+    execute: LocalLabPhaseExecute<T>;
+}
+interface LocalLabPhaseOutcome<T> {
+    phase: LocalLabPhaseDescriptor;
+    preflight: LocalLabPreflightResult;
+    result: T;
+}
+interface SequentialPhaseHooks {
+    /**
+     * Called after a phase's preflight succeeds but before its `execute`.
+     * Useful for logging "phase X starting against <endpoint>".
+     */
+    onPhasePreflight?: (result: LocalLabPreflightResult) => void;
+    /** Called immediately before invoking the phase's execute callback. */
+    onPhaseStart?: (phase: LocalLabPhaseDescriptor) => void;
+    /** Called immediately after a phase's execute resolves. */
+    onPhaseComplete?: <T>(outcome: LocalLabPhaseOutcome<T>) => void;
+    /**
+     * Called between two phases when their endpoints differ. Receives the
+     * manifest hand-off note (or undefined). The scheduler does NOT print to
+     * stdout itself — it delegates to this hook so tests can observe it
+     * without capturing process output.
+     */
+    onPhaseHandoff?: (from: LocalLabPhaseDescriptor, to: LocalLabPhaseDescriptor, note: string | undefined) => void;
+}
+interface RunSequentialPhasesOptions {
+    /** Forwarded to each phase's preflight. */
+    preflight?: LocalLabPreflightOptions;
+    hooks?: SequentialPhaseHooks;
+}
+/**
+ * Run a sequence of phases against the manifest's declared endpoints, with
+ * preflight + hand-off enforcement between them. Resolves with each phase's
+ * outcome in order, or rejects with a `LocalLabPreflightError`
+ * carrying the failing preflight (which surfaces the endpoint's found model
+ * list per rule 51).
+ *
+ * Phases are required to be supplied in run order; the scheduler does not
+ * reorder them. Empty phase lists are a no-op (the harness intentionally
+ * supports an empty responder/judge pair for preflight-only smoke checks).
+ */
+declare function runSequentialPhases<T>(manifest: LocalLabManifest, phases: LocalLabPhase<T>[], options?: RunSequentialPhasesOptions): Promise<LocalLabPhaseOutcome<T>[]>;
+/**
+ * Error thrown when a phase's preflight fails. Carries the failing result
+ * so callers (and tests) can inspect what the endpoint reported.
+ */
+declare class LocalLabPreflightError extends Error {
+    readonly phase: LocalLabPhaseDescriptor;
+    readonly preflight: LocalLabPreflightResult;
+    readonly phaseIndex: number;
+    constructor(args: {
+        phase: LocalLabPhaseDescriptor;
+        preflight: LocalLabPreflightResult;
+        phaseIndex: number;
+    });
+}
+/**
+ * Render the operator hand-off string the scheduler passes to the
+ * `onPhaseHandoff` hook. Returns the manifest note when authored, otherwise
+ * a default instruction naming the next phase and its endpoint. This is a
+ * pure formatting helper — tests assert on it directly.
+ */
+declare function formatHandoffNote(from: LocalLabPhaseDescriptor, to: LocalLabPhaseDescriptor, manifestNote: string | undefined): string;
+
 type BenchModelSource = "plugin" | "gateway";
 interface ResolveBenchRuntimeProfileOptions {
     runtimeProfile?: BenchRuntimeProfile;
@@ -1754,6 +2112,12 @@ interface ResolveBenchRuntimeProfileOptions {
     drainTimeout?: number;
     max429WaitMs?: number;
     disableThinking?: boolean;
+    /**
+     * Path to a local-lab manifest JSON file (issue #1573 PR2). Required when
+     * `runtimeProfile: "local-lab"`. The manifest pins responder/judge/embedding
+     * to operator-hosted models with temperature=0 and a fixed seed.
+     */
+    localLabManifestPath?: string;
 }
 interface ResolvedBenchRuntimeProfile {
     profile: BenchRuntimeProfile;
@@ -1769,6 +2133,12 @@ interface ResolvedBenchRuntimeProfile {
     systemProvider: ProviderConfig | null;
     judgeProvider: ProviderConfig | null;
     internalProvider: ProviderConfig | null;
+    /**
+     * Resolved local-lab profile (issue #1573 PR2). Present only when
+     * `runtimeProfile: "local-lab"`. Drives sequential phase scheduling +
+     * endpoint preflight in the bench runner.
+     */
+    localLab?: ResolvedLocalLabProfile;
 }
 declare function resolveBenchRuntimeProfile(options: ResolveBenchRuntimeProfileOptions): Promise<ResolvedBenchRuntimeProfile>;
 
@@ -3188,4 +3558,4 @@ interface MitigatedTargetConfig {
  */
 declare function createMitigatedTarget(config: MitigatedTargetConfig): ExtractionAttackTarget;
 
-export { AMA_BENCH_DIAGNOSTIC_VARIANTS, ASSISTANT_AGENT_CONFIG_KEY, ASSISTANT_JUDGE_CONFIG_KEY, ASSISTANT_MEETING_PREP_SCENARIOS, ASSISTANT_MEETING_PREP_SMOKE_SCENARIOS, ASSISTANT_MORNING_BRIEF_SCENARIOS, ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SMOKE_SCENARIOS, ASSISTANT_RUBRIC_DIMENSIONS, ASSISTANT_RUBRIC_ID_KEY, ASSISTANT_SEEDS_CONFIG_KEY, ASSISTANT_SPOT_CHECK_DIR_KEY, ASSISTANT_SYNTHESIS_SCENARIOS, ASSISTANT_SYNTHESIS_SMOKE_SCENARIOS, type AbstentionRetrievalCase, type AggregateMetrics, type AmaBenchDiagnosticAdapterOptions, type AmaBenchDiagnosticAnswererMode, type AmaBenchDiagnosticBreakdown, type AmaBenchDiagnosticMatrixArtifact, type AmaBenchDiagnosticRecallMode, type AmaBenchDiagnosticRunContext, type AmaBenchDiagnosticTaskEvidence, type AmaBenchDiagnosticTaskRow, type AmaBenchDiagnosticVariant, type AmaBenchDiagnosticVariantSummary, type AnthropicProviderConfig, type AssistantAgent, type AssistantMemoryFact, type AssistantMemoryGraph, type AssistantRubricDimension, type AssistantRubricScores, type AssistantRunnerOptions, type AssistantScenario, type AssistantStance, type AttackRecallOptions, type AttackRetrievalHit, type AttackerMode, BENCHMARK_ARTIFACT_SCHEMA_VERSION, BENCHMARK_INTEGRITY_META_SCHEMA, BENCHMARK_REPRO_MANIFEST_FILENAME, BENCHMARK_REPRO_MANIFEST_SCHEMA_VERSION, BENCHMARK_RESULT_SCHEMA, BENCHMARK_SPLIT_TYPES, type BaselineRow, type BaselineScenario, type BeamDatasetPreview, type BenchConfig, type BenchJudge, type BenchJudgeResult, type BenchMemoryAdapter, type BenchModelSource, type BenchReasoningEffort, type BenchRecallOptions, type BenchResponder, type BenchResponse, type BenchRuntimeProfile, type BenchTier, type BenchmarkArtifact, type BenchmarkArtifactEnvironment, type BenchmarkArtifactPerTaskScore, type BenchmarkArtifactSystem, type BenchmarkCategory, type BenchmarkDefinition, type BenchmarkIntegrityMeta, type BenchmarkMeta, type BenchmarkMode, type BenchmarkReport, type BenchmarkReproManifest, type BenchmarkReproManifestDataset, type BenchmarkReproManifestFile, type BenchmarkReproManifestResult, type BenchmarkResult, type BenchmarkSplitType, type BenchmarkStatus, type BenchmarkSuiteResult, type BenchmarkTier, type BuildBenchmarkArtifactInput, type BuildBenchmarkPublishFeedOptions, type BuildBenchmarkReproManifestOptions, type BuiltInProvider, CANARY_FIXED_RECALL, CANARY_SCORE_FLOOR, type CanaryAdapterOptions, type CanaryFloorCheck, type CodexCliProviderConfig, type ComparisonMetricDelta, type ComparisonResult, type CompletionOpts, type CompletionResult, type ConfidenceInterval, type ContaminationCheckResult, type ContaminationEntry, type ContaminationManifest, type CustomBenchmarkScoring, type CustomBenchmarkSpec, type CustomBenchmarkTask, DEFAULT_ABLATION_BOOTSTRAP_SEED, DEFAULT_ASSISTANT_RUBRIC_ID, DEFAULT_BASELINE_SCENARIOS, type DatasetSource, type DiscoveredModel, EMPTY_CONTAMINATION_MANIFEST, type EffectSizeInterpretation, type EffectSizeSummary, type ExplainResult, type ExtractedEntity, type ExtractedLink, type ExtractedPage, type ExtractionAttackOptions, type ExtractionAttackResult, type ExtractionAttackTarget, type FixtureGenerator, type FixtureOutput, type FixtureVariant, type GeneratedFile, type GoldEntity, type GoldEntityType, type GoldGraph, type GoldLink, type GoldPage, type HarnessRng, INTEGRITY_CIPHER_ALGORITHM, INTEGRITY_HASH_ALGORITHM, INTEGRITY_META_FIELDS, type IngestionBenchAdapter, type IngestionLog, LOCOMO_DATASET_FILENAMES, LONG_MEM_EVAL_DATASET_FILENAMES, type LeaderboardArtifactWrite, type LlmJudge, type LlmProvider, type LoadDatasetOptions, type LoadSealedQrelsOptions, type LoadedDataset, type LocalLlmProviderConfig, MEMORY_EVAL_DIMENSIONS, MEMORY_EVAL_PUBLIC_LINE, MITIGATED_BASELINE_SCENARIOS, type MemoryEvalCategory, type MemoryEvalDimension, type MemoryEvalDimensionId, type MemoryEvalMetric, type MemoryGraph, type MemoryStats, type MemorySystem, type Message, type MetricAggregate, type MitigatedBaselineConfig, type MitigatedTargetConfig, type MultipleChoiceQuestion, OTHER_NAMESPACE_MEMORIES, type OllamaProviderConfig, type OpenAiCompatibleProviderConfig, PROCEDURAL_REAL_SCENARIOS, PROCEDURAL_REAL_SCENARIOS_SMOKE, PUBLISHED_BENCHMARK_ARTIFACT_IDS, type PersonalizationRetrievalCase, type ProceduralAblationArtifact, type ProceduralAblationPerCase, type ProceduralAblationScenario, type ProceduralRealScenario, type ProceduralRealScenarioCategory, type ProviderBaseConfig, type ProviderConfig, type ProviderDiscoveryResult, type ProviderFactoryConfig, type PublishSkipReason, type PublishSkipRecord, type PublishedBenchmarkFeed, type PublishedBenchmarkFeedEntry, type PublishedBenchmarkId, REQUIRED_FRONTMATTER_FIELDS, type RecallMetrics, type RecoveredMemory, type RegressionDetail, type RegressionGateResult, type RemnicAdapterOptions, type ResolveBenchRuntimeProfileOptions, type ResolvedBenchRuntimeProfile, type ResolvedRunBenchmarkOptions, type RotatedChoices, type RunBenchmarkOptions, type RunProceduralAblationCliArgs, type RunProceduralAblationOptions, SCHEMA_TIER_FIXTURE, SCHEMA_TIER_SMOKE_FIXTURE, SEALED_PROMPT_REGISTRY, SYNTHETIC_MEMORIES, type SanitizedDiagnosticProvider, type SavedBaseline, type SchemaTierCorpus, type SchemaTierFixture, type SchemaTierName, type SchemaTierPage, type SchemaTierPageFrontmatter, type SealedArtifact, type SealedJudgeDecision, type SealedJudgeInput, type SealedQrelsArtifact, type SealedQrelsHandle, type SealedRubric, type SearchResult, type SeededMemory, type SeededRng, type SpotCheckLogger, type StatisticalReport, type StructuredJudge, type SyntheticEmailIngestionAdapterOptions, type SyntheticTargetOptions, type TaskResult, type TaskTokenUsage, type TemporalRetrievalCase, type TierDetail, type TimelineEntry, type TokenUsage, type WriteBenchmarkArtifactResult, addContaminationEntry, aggregateTaskScores, answerBenchmarkQuestion, assertCanaryUnderFloor, assertIntegrityMetaPresent, assertPublishableIntegrity, assertSha256Hex, assistantMeetingPrepDefinition, assistantMorningBriefDefinition, assistantNextBestActionDefinition, assistantSynthesisDefinition, backlinkF1, bootstrapMeanConfidenceInterval, buildAmaBenchDiagnosticMatrixArtifact, buildAmaBenchDiagnosticVariantSummary, buildAmaBenchLeaderboardRows, buildBenchmarkArtifact, buildBenchmarkArtifactFilename, buildBenchmarkPublishFeed, buildBenchmarkReproManifest, buildBenchmarkRunSeeds, buildJudgePayload, buildOracleTrajectoryRecall, buildSchemaTierFixture, buildSchemaTierSmokeFixture, calendarFixture, canonicalJsonStringify, chatFixture, checkDatasetContamination, checkRegression, clampScore, cohensD, compareResults, computeSealHash, containsAnswer, createSeededRng as createAdamSeededRng, createAmaBenchDiagnosticAdapter, createAnthropicProvider, createCanaryAdapter, createCodexCliProvider, createDeterministicSpotCheckLogger, createGatewayResponder, createLightweightAdapter, createLiteLlmProvider, createLocalLlmProvider, createMitigatedTarget, createOllamaProvider, createOpenAiCompatibleProvider, createSeededRandom as createProceduralAblationSeededRandom, createProvider, createProviderBackedAmaBenchRecommendedJudge, createProviderBackedJudge, createProviderBackedResponder, createProviderBackedStructuredJudge, createRemnicAdapter, createResponderFromProvider, createSeededRng$1 as createSeededRng, createSpotCheckFileLogger, createStructuredJudgeFromProvider, createSyntheticEmailIngestionAdapter, createSyntheticTarget, createTimeoutGuardedAdapter, defaultBenchmarkBaselineDir, defaultBenchmarkPublishPath, deleteBenchmarkResults, discoverAllProviders, emailFixture, entityRecall, exactMatch, extractMarkdownSectionsByTitle, f1Score, fixtureToAblationScenarios, formatMissingDatasetError, generateReport, getBenchmark, getBenchmarkLowerIsBetter, getMemoryEvalDimension, getRemnicVersion, hashBenchmarkArtifact, hashBytes, hashCanonicalJson, hashString, integrityMetaIsComplete, interpretEffectSize, isAmaBenchUnknownLikeAnswer, isContaminationEntry, isContaminationManifest, isSealedQrelsArtifact, isSha256Hex, linkMatches, listBenchmarkBaselines, listBenchmarkResults, listBenchmarks, listMemoryEvalBenchmarkIds, listMemoryEvalDimensions, llmJudgeScore, llmJudgeScoreDetailed, loadAblationFixture, loadBaseline, loadBeamDatasetPreview, loadBenchmarkArtifact, loadBenchmarkBaseline, loadBenchmarkResult, loadCustomBenchmarkFile, loadLoCoMo10, loadLongMemEvalS, loadSealKeyFromEnv, loadSealedQrels, loadSealedRubric, matchEntity, mergeContaminationManifests, openSeal, orchestrateBenchmarkRuns, pairedDeltaConfidenceInterval, parseBenchmarkArtifact, parseCustomBenchmark, parseRubricResponse, parseSealedQrels, precisionAtK, projectFolderFixture, recallAtK, redactBenchmarkResultSecrets, renderBaselineMarkdown, renderBenchmarkResultExport, renderMemorySummaryForJudge, renderMemoryViewForAgent, resolveAssistantAgent, resolveAssistantRubricId, resolveAssistantSeeds, resolveAssistantSpotCheckDir, resolveBenchRuntimeProfile, resolveBenchmarkPhaseTimeoutMs, resolveBenchmarkProgressLogging, resolveBenchmarkResultReference, resolveBenchmarkRunCount, resolveStructuredJudge, rotateDistractors, rougeL, runAssistantBenchmark, runAssistantMeetingPrepBenchmark, runAssistantMorningBriefBenchmark, runAssistantNextBestActionBenchmark, runAssistantSynthesisBenchmark, runBaseline, runBenchSuite, runBenchmark, runCustomBenchmarkFile, runExplain, runExtractionAttack, runMitigatedBaseline, runProceduralAblation, runProceduralAblationCli, runSealedJudge, safeHexEqual, saveBaseline, saveBenchmarkBaseline, schemaCompleteness, sealPayload, selectAmaBenchDiagnosticVariants, selectFixtureVariant, serializeBenchmarkArtifact, serializeJsonl, serializeSealedQrels, shuffleTasks, timed, verifyRubricDigest, writeBenchmarkArtifact, writeBenchmarkPublishFeed, writeBenchmarkReproManifest, writeBenchmarkResult, writeLeaderboardArtifactsForResult, zeroScores };
+export { AMA_BENCH_DIAGNOSTIC_VARIANTS, ASSISTANT_AGENT_CONFIG_KEY, ASSISTANT_JUDGE_CONFIG_KEY, ASSISTANT_MEETING_PREP_SCENARIOS, ASSISTANT_MEETING_PREP_SMOKE_SCENARIOS, ASSISTANT_MORNING_BRIEF_SCENARIOS, ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SCENARIOS, ASSISTANT_NEXT_BEST_ACTION_SMOKE_SCENARIOS, ASSISTANT_RUBRIC_DIMENSIONS, ASSISTANT_RUBRIC_ID_KEY, ASSISTANT_SEEDS_CONFIG_KEY, ASSISTANT_SPOT_CHECK_DIR_KEY, ASSISTANT_SYNTHESIS_SCENARIOS, ASSISTANT_SYNTHESIS_SMOKE_SCENARIOS, type AbstentionRetrievalCase, type AggregateMetrics, type AmaBenchDiagnosticAdapterOptions, type AmaBenchDiagnosticAnswererMode, type AmaBenchDiagnosticBreakdown, type AmaBenchDiagnosticMatrixArtifact, type AmaBenchDiagnosticRecallMode, type AmaBenchDiagnosticRunContext, type AmaBenchDiagnosticTaskEvidence, type AmaBenchDiagnosticTaskRow, type AmaBenchDiagnosticVariant, type AmaBenchDiagnosticVariantSummary, type AnthropicProviderConfig, type AssistantAgent, type AssistantMemoryFact, type AssistantMemoryGraph, type AssistantRubricDimension, type AssistantRubricScores, type AssistantRunnerOptions, type AssistantScenario, type AssistantStance, type AttackRecallOptions, type AttackRetrievalHit, type AttackerMode, BENCHMARK_ARTIFACT_SCHEMA_VERSION, BENCHMARK_INTEGRITY_META_SCHEMA, BENCHMARK_REPRO_MANIFEST_FILENAME, BENCHMARK_REPRO_MANIFEST_SCHEMA_VERSION, BENCHMARK_RESULT_SCHEMA, BENCHMARK_SPLIT_TYPES, type BaselineRow, type BaselineScenario, type BeamDatasetPreview, type BenchConfig, type BenchJudge, type BenchJudgeResult, type BenchMemoryAdapter, type BenchModelSource, type BenchReasoningEffort, type BenchRecallOptions, type BenchResponder, type BenchResponse, type BenchRuntimeProfile, type BenchTier, type BenchmarkArtifact, type BenchmarkArtifactEnvironment, type BenchmarkArtifactPerTaskScore, type BenchmarkArtifactSystem, type BenchmarkCategory, type BenchmarkDefinition, type BenchmarkIntegrityMeta, type BenchmarkMeta, type BenchmarkMode, type BenchmarkReport, type BenchmarkReproManifest, type BenchmarkReproManifestDataset, type BenchmarkReproManifestFile, type BenchmarkReproManifestResult, type BenchmarkResult, type BenchmarkSplitType, type BenchmarkStatus, type BenchmarkSuiteResult, type BenchmarkTier, type BuildBenchmarkArtifactInput, type BuildBenchmarkPublishFeedOptions, type BuildBenchmarkReproManifestOptions, type BuiltInProvider, CANARY_FIXED_RECALL, CANARY_SCORE_FLOOR, type CanaryAdapterOptions, type CanaryFloorCheck, type CodexCliProviderConfig, type ComparisonMetricDelta, type ComparisonResult, type CompletionOpts, type CompletionResult, type ConfidenceInterval, type ContaminationCheckResult, type ContaminationEntry, type ContaminationManifest, type CustomBenchmarkScoring, type CustomBenchmarkSpec, type CustomBenchmarkTask, DEFAULT_ABLATION_BOOTSTRAP_SEED, DEFAULT_ASSISTANT_RUBRIC_ID, DEFAULT_BASELINE_SCENARIOS, type DatasetSource, type DiscoveredModel, EMPTY_CONTAMINATION_MANIFEST, type EffectSizeInterpretation, type EffectSizeSummary, type ExplainResult, type ExtractedEntity, type ExtractedLink, type ExtractedPage, type ExtractionAttackOptions, type ExtractionAttackResult, type ExtractionAttackTarget, type FixtureGenerator, type FixtureOutput, type FixtureVariant, type GeneratedFile, type GoldEntity, type GoldEntityType, type GoldGraph, type GoldLink, type GoldPage, type HarnessRng, INTEGRITY_CIPHER_ALGORITHM, INTEGRITY_HASH_ALGORITHM, INTEGRITY_META_FIELDS, type IngestionBenchAdapter, type IngestionLog, LOCAL_LAB_PROVIDER_KINDS, LOCOMO_DATASET_FILENAMES, LONG_MEM_EVAL_DATASET_FILENAMES, type LeaderboardArtifactWrite, type LlmJudge, type LlmProvider, type LoadDatasetOptions, type LoadSealedQrelsOptions, type LoadedDataset, type LocalLabManifest, type LocalLabManifestNotes, type LocalLabPhase, type LocalLabPhaseDescriptor, type LocalLabPhaseExecute, type LocalLabPhaseName, type LocalLabPhaseOutcome, LocalLabPreflightError, type LocalLabPreflightFailure, type LocalLabPreflightInput, type LocalLabPreflightOptions, type LocalLabPreflightResult, type LocalLabPreflightSuccess, type LocalLabProviderKind, type LocalLabRoleConfig, type LocalLlmProviderConfig, MEMORY_EVAL_DIMENSIONS, MEMORY_EVAL_PUBLIC_LINE, MITIGATED_BASELINE_SCENARIOS, type MemoryEvalCategory, type MemoryEvalDimension, type MemoryEvalDimensionId, type MemoryEvalMetric, type MemoryGraph, type MemoryStats, type MemorySystem, type Message, type MetricAggregate, type MitigatedBaselineConfig, type MitigatedTargetConfig, type MultipleChoiceQuestion, OTHER_NAMESPACE_MEMORIES, type OllamaProviderConfig, type OpenAiCompatibleProviderConfig, PROCEDURAL_REAL_SCENARIOS, PROCEDURAL_REAL_SCENARIOS_SMOKE, PUBLISHED_BENCHMARK_ARTIFACT_IDS, type PersonalizationRetrievalCase, type PreflightDiscoveredModel, type ProceduralAblationArtifact, type ProceduralAblationPerCase, type ProceduralAblationScenario, type ProceduralRealScenario, type ProceduralRealScenarioCategory, type ProviderBaseConfig, type ProviderConfig, type ProviderDiscoveryResult, type ProviderFactoryConfig, type PublishSkipReason, type PublishSkipRecord, type PublishedBenchmarkFeed, type PublishedBenchmarkFeedEntry, type PublishedBenchmarkId, REQUIRED_FRONTMATTER_FIELDS, type RecallMetrics, type RecoveredMemory, type RegressionDetail, type RegressionGateResult, type RemnicAdapterOptions, type ResolveBenchRuntimeProfileOptions, type ResolvedBenchRuntimeProfile, type ResolvedLocalLabProfile, type ResolvedLocalLabRole, type ResolvedRunBenchmarkOptions, type RotatedChoices, type RunBenchmarkOptions, type RunProceduralAblationCliArgs, type RunProceduralAblationOptions, type RunSequentialPhasesOptions, SCHEMA_TIER_FIXTURE, SCHEMA_TIER_SMOKE_FIXTURE, SEALED_PROMPT_REGISTRY, SYNTHETIC_MEMORIES, type SanitizedDiagnosticProvider, type SavedBaseline, type SchemaTierCorpus, type SchemaTierFixture, type SchemaTierName, type SchemaTierPage, type SchemaTierPageFrontmatter, type SealedArtifact, type SealedJudgeDecision, type SealedJudgeInput, type SealedQrelsArtifact, type SealedQrelsHandle, type SealedRubric, type SearchResult, type SeededMemory, type SeededRng, type SequentialPhaseHooks, type SpotCheckLogger, type StatisticalReport, type StructuredJudge, type SyntheticEmailIngestionAdapterOptions, type SyntheticTargetOptions, type TaskResult, type TaskTokenUsage, type TemporalRetrievalCase, type TierDetail, type TimelineEntry, type TokenUsage, type WriteBenchmarkArtifactResult, addContaminationEntry, aggregateTaskScores, answerBenchmarkQuestion, assertCanaryUnderFloor, assertIntegrityMetaPresent, assertPublishableIntegrity, assertSha256Hex, assistantMeetingPrepDefinition, assistantMorningBriefDefinition, assistantNextBestActionDefinition, assistantSynthesisDefinition, backlinkF1, bootstrapMeanConfidenceInterval, buildAmaBenchDiagnosticMatrixArtifact, buildAmaBenchDiagnosticVariantSummary, buildAmaBenchLeaderboardRows, buildBenchmarkArtifact, buildBenchmarkArtifactFilename, buildBenchmarkPublishFeed, buildBenchmarkReproManifest, buildBenchmarkRunSeeds, buildJudgePayload, buildOracleTrajectoryRecall, buildSchemaTierFixture, buildSchemaTierSmokeFixture, calendarFixture, canonicalJsonStringify, chatFixture, checkDatasetContamination, checkRegression, clampScore, cohensD, compareResults, computeSealHash, containsAnswer, createSeededRng as createAdamSeededRng, createAmaBenchDiagnosticAdapter, createAnthropicProvider, createCanaryAdapter, createCodexCliProvider, createDeterministicSpotCheckLogger, createGatewayResponder, createLightweightAdapter, createLiteLlmProvider, createLocalLlmProvider, createMitigatedTarget, createOllamaProvider, createOpenAiCompatibleProvider, createSeededRandom as createProceduralAblationSeededRandom, createProvider, createProviderBackedAmaBenchRecommendedJudge, createProviderBackedJudge, createProviderBackedResponder, createProviderBackedStructuredJudge, createRemnicAdapter, createResponderFromProvider, createSeededRng$1 as createSeededRng, createSpotCheckFileLogger, createStructuredJudgeFromProvider, createSyntheticEmailIngestionAdapter, createSyntheticTarget, createTimeoutGuardedAdapter, defaultBenchmarkBaselineDir, defaultBenchmarkPublishPath, deleteBenchmarkResults, discoverAllProviders, discoveryEndpointFor, emailFixture, entityRecall, exactMatch, extractMarkdownSectionsByTitle, f1Score, fixtureToAblationScenarios, formatHandoffNote, formatMissingDatasetError, generateReport, getBenchmark, getBenchmarkLowerIsBetter, getMemoryEvalDimension, getRemnicVersion, hashBenchmarkArtifact, hashBytes, hashCanonicalJson, hashString, integrityMetaIsComplete, interpretEffectSize, isAmaBenchUnknownLikeAnswer, isContaminationEntry, isContaminationManifest, isSealedQrelsArtifact, isSha256Hex, linkMatches, listBenchmarkBaselines, listBenchmarkResults, listBenchmarks, listMemoryEvalBenchmarkIds, listMemoryEvalDimensions, llmJudgeScore, llmJudgeScoreDetailed, loadAblationFixture, loadBaseline, loadBeamDatasetPreview, loadBenchmarkArtifact, loadBenchmarkBaseline, loadBenchmarkResult, loadCustomBenchmarkFile, loadLoCoMo10, loadLocalLabManifest, loadLongMemEvalS, loadSealKeyFromEnv, loadSealedQrels, loadSealedRubric, matchEntity, mergeContaminationManifests, openSeal, orchestrateBenchmarkRuns, pairedDeltaConfidenceInterval, parseBenchmarkArtifact, parseCustomBenchmark, parseLocalLabManifest, parseRubricResponse, parseSealedQrels, precisionAtK, preflightLocalLabRole, projectFolderFixture, recallAtK, redactBenchmarkResultSecrets, renderBaselineMarkdown, renderBenchmarkResultExport, renderMemorySummaryForJudge, renderMemoryViewForAgent, resolveAssistantAgent, resolveAssistantRubricId, resolveAssistantSeeds, resolveAssistantSpotCheckDir, resolveBenchRuntimeProfile, resolveBenchmarkPhaseTimeoutMs, resolveBenchmarkProgressLogging, resolveBenchmarkResultReference, resolveBenchmarkRunCount, resolveLocalLabProfile, resolveLocalLabRole, resolveStructuredJudge, rotateDistractors, rougeL, runAssistantBenchmark, runAssistantMeetingPrepBenchmark, runAssistantMorningBriefBenchmark, runAssistantNextBestActionBenchmark, runAssistantSynthesisBenchmark, runBaseline, runBenchSuite, runBenchmark, runCustomBenchmarkFile, runExplain, runExtractionAttack, runMitigatedBaseline, runProceduralAblation, runProceduralAblationCli, runSealedJudge, runSequentialPhases, safeHexEqual, saveBaseline, saveBenchmarkBaseline, schemaCompleteness, sealPayload, selectAmaBenchDiagnosticVariants, selectFixtureVariant, serializeBenchmarkArtifact, serializeJsonl, serializeSealedQrels, shuffleTasks, timed, verifyRubricDigest, writeBenchmarkArtifact, writeBenchmarkPublishFeed, writeBenchmarkReproManifest, writeBenchmarkResult, writeLeaderboardArtifactsForResult, zeroScores };