codemem 0.44.0-beta.1 → 0.44.0

package/dist/index.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { DEDUP_KEY_BACKFILL_JOB, DEFAULT_COORDINATOR_DB_PATH, DedupKeyBackfillRunner, DeviceIdentityError, MUTATING_TOOL_NAMES, MemoryStore, ObserverClient, PROMPT_TRANSPORT_PROTOCOL_RANGE, REF_BACKFILL_JOB, RawEventIngestValidationError, RawEventSweeper, RefBackfillRunner, SCOPE_BACKFILL_JOB, SESSION_CONTEXT_BACKFILL_JOB, SUMMARY_DEDUP_BACKFILL_JOB, ScopeBackfillRunner, SessionContextBackfillRunner, SummaryDedupBackfillRunner, SyncRetentionRunner, TRUSTED_HOOK_MAPPER_OPTIONS, VERSION, VectorModelMigrationRunner, aiBackfillStructuredContent, applyBootstrapSnapshot, applyDistillRule, arePromptTransportProtocolRangesCompatible, backfillMemoryDedupKeys, backfillNarrativeFromBody, backfillTagsText, backfillVectors, buildBaseUrl, buildDirectPeerAuthHeaders, buildDistillReport, buildRawEventEnvelopeFromCodexHook, buildRawEventEnvelopeFromHook, buildViewerIdentityTarget, classifyPromptTransportFailure, clonePromptPackAttempt, collectOperationalStatus, compareMemoryRoleReports, connect, connectReadOnly, coordinatorCreateGroupAction, coordinatorCreateInviteAction, coordinatorCreateScopeAction, coordinatorDisableDeviceAction, coordinatorEnrollDeviceAction, coordinatorGrantScopeMembershipAction, coordinatorImportInviteAction, coordinatorListBootstrapGrantsAction, coordinatorListDevicesAction, coordinatorListGroupsAction, coordinatorListJoinRequestsAction, coordinatorListScopeMembershipsAction, coordinatorListScopesAction, coordinatorRemoveDeviceAction, coordinatorRenameDeviceAction, coordinatorReviewJoinRequestAction, coordinatorRevokeBootstrapGrantAction, coordinatorRevokeScopeMembershipAction, coordinatorUpdateScopeAction, createBetterSqliteCoordinatorApp, deactivateLowSignalMemories, deactivateLowSignalObservations, dedupNearDuplicateMemories, detectInstallKind, draftDistillRule, ensureDeviceIdentity, ensureSchemaBootstrapped, estimateExtractionModelCost, exportMemories, extractApplyPatchPaths, fetchAllSnapshotPages, fingerprintPublicKey, flushRawEvents, formatHostPort, getAttributionDiagnostics, getExtractionBenchmarkProfile, getExtractionModelPricing, getInjectionEvalScenarioPack, getInjectionEvalScenarioPrompts, getMaintenanceJob, getMemoryArtifactReport, getMemoryRoleReport, getRawEventRelinkPlan, getRawEventRelinkReport, getRawEventStatus, getSemanticIndexDiagnostics, getSessionExtractionEval, getSessionExtractionEvalScenario, getUpdateStatus, getWorkspaceCodememConfigPath, hasPendingDedupKeyBackfill, hasPendingRefBackfill, hasPendingScopeBackfill, hasPendingSessionContextBackfill, hasPendingSummaryDedupBackfill, hasUnsyncedSharedMemoryChanges, importMemories, ingestRawEvents, initDatabase, isEmbeddingDisabled, isReleaseVersionForChannel, judgeDistillReport, listMaintenanceJobs, listPerPeerScopeSyncState, listRetentionScopeIds, loadObserverConfig, loadPublicKey, loadSqliteVec, mdnsEnabled, normalizePromptTransportProtocolRange, planReplicationOpsAgePrune, probeCodememViewerLiveness, probeCodememViewerLiveness as probeCodememViewerLiveness$1, probeRequiredNativeRuntime, projectMatchesFilter, promptPackArtifactFingerprint, pruneReplicationOpsUntilCaughtUp, rawEventsGate, readCodememConfigFile, readCodememConfigFileAtPath, readCoordinatorSyncConfig, readImportPayload, recordPromptPackArtifacts, recordPromptPackTerminal, recordRetrievalSurface, renderUnifiedDiff, replayBatchExtraction, replayBatchExtractionWithTierRouting, requestJson, resolveCodememConfigPath, resolveDbPath, resolveHookProject, resolveProject, resolveProjectRoot, resolveRetrievalSession, retryRawEventFailures, runSyncDaemon, runSyncPass, scanSecretsRetroactive, schema, scoreExtractionBenchmarkOutput, setPeerProjectFilter, stripJsonComments, stripTrailingCommas, syncPassPreflight, tryUpdateRetrievalDelivery, updatePeerAddresses, vacuumDatabase, viewerUrl, writeCodememConfigFile } from "@codemem/core";
+import { DEDUP_KEY_BACKFILL_JOB, DEFAULT_COORDINATOR_DB_PATH, DedupKeyBackfillRunner, DeviceIdentityError, EXTRACTION_BENCHMARK_QUALITY_WEIGHTS, MUTATING_TOOL_NAMES, MemoryStore, ObserverClient, ObserverOutputError, ObserverOutputTransportError, PROMPT_TRANSPORT_PROTOCOL_RANGE, REF_BACKFILL_JOB, RawEventIngestValidationError, RawEventSweeper, RefBackfillRunner, SCOPE_BACKFILL_JOB, SESSION_CONTEXT_BACKFILL_JOB, SUMMARY_DEDUP_BACKFILL_JOB, ScopeBackfillRunner, SessionContextBackfillRunner, SummaryDedupBackfillRunner, SyncRetentionRunner, TRUSTED_HOOK_MAPPER_OPTIONS, VERSION, VectorModelMigrationRunner, aiBackfillStructuredContent, applyBootstrapSnapshot, applyDistillRule, arePromptTransportProtocolRangesCompatible, backfillMemoryDedupKeys, backfillNarrativeFromBody, backfillTagsText, backfillVectors, buildBaseUrl, buildDirectPeerAuthHeaders, buildDistillReport, buildRawEventEnvelopeFromCodexHook, buildRawEventEnvelopeFromHook, buildViewerIdentityTarget, classifyPromptTransportFailure, clonePromptPackAttempt, collectOperationalStatus, compareMemoryRoleReports, connect, connectReadOnly, coordinatorCreateGroupAction, coordinatorCreateInviteAction, coordinatorCreateScopeAction, coordinatorDisableDeviceAction, coordinatorEnrollDeviceAction, coordinatorGrantScopeMembershipAction, coordinatorImportInviteAction, coordinatorListBootstrapGrantsAction, coordinatorListDevicesAction, coordinatorListGroupsAction, coordinatorListJoinRequestsAction, coordinatorListScopeMembershipsAction, coordinatorListScopesAction, coordinatorRemoveDeviceAction, coordinatorRenameDeviceAction, coordinatorReviewJoinRequestAction, coordinatorRevokeBootstrapGrantAction, coordinatorRevokeScopeMembershipAction, coordinatorUpdateScopeAction, createBetterSqliteCoordinatorApp, deactivateLowSignalMemories, deactivateLowSignalObservations, dedupNearDuplicateMemories, detectInstallKind, draftDistillRule, ensureDeviceIdentity, ensureSchemaBootstrapped, estimateExtractionModelCost, exportMemories, extractApplyPatchPaths, extractionReplayObserverIdentity, fetchAllSnapshotPages, fingerprintPublicKey, flushRawEvents, formatHostPort, getAttributionDiagnostics, getExtractionBenchmarkProfile, getExtractionModelPricing, getInjectionEvalScenarioPack, getInjectionEvalScenarioPrompts, getMaintenanceJob, getMemoryArtifactReport, getMemoryRoleReport, getRawEventRelinkPlan, getRawEventRelinkReport, getRawEventStatus, getSemanticIndexDiagnostics, getSessionExtractionEval, getSessionExtractionEvalScenario, getUpdateStatus, getWorkspaceCodememConfigPath, hasPendingDedupKeyBackfill, hasPendingRefBackfill, hasPendingScopeBackfill, hasPendingSessionContextBackfill, hasPendingSummaryDedupBackfill, hasUnsyncedSharedMemoryChanges, importMemories, ingestRawEvents, initDatabase, isAutomaticRecallMeasurement, isEmbeddingDisabled, isReleaseVersionForChannel, judgeDistillReport, listMaintenanceJobs, listPerPeerScopeSyncState, listRetentionScopeIds, loadObserverConfig, loadPublicKey, loadSqliteVec, mdnsEnabled, normalizePromptTransportProtocolRange, planReplicationOpsAgePrune, probeCodememViewerLiveness, probeCodememViewerLiveness as probeCodememViewerLiveness$1, probeRequiredNativeRuntime, projectMatchesFilter, promptPackArtifactFingerprint, pruneReplicationOpsUntilCaughtUp, rawEventsGate, readCodememConfigFile, readCodememConfigFileAtPath, readCoordinatorSyncConfig, readImportPayload, recordAutomaticRecall, recordPromptPackArtifacts, recordPromptPackTerminal, recordRetrievalSurface, renderUnifiedDiff, replayBatchExtraction, replayBatchExtractionWithTierRouting, requestJson, resolveCodememConfigPath, resolveDbPath, resolveHookProject, resolveProject, resolveProjectRoot, resolveRetrievalSession, retryRawEventFailures, runSyncDaemon, runSyncPass, scanSecretsRetroactive, schema, scoreExtractionBenchmarkOutput, setPeerProjectFilter, stripJsonComments, stripTrailingCommas, syncPassPreflight, tryUpdateRetrievalDelivery, updatePeerAddresses, vacuumDatabase, viewerUrl, writeCodememConfigFile } from "@codemem/core";
 import { Command, Option } from "commander";
 import omelette from "omelette";
 import { randomInt, randomUUID } from "node:crypto";
@@ -5217,6 +5217,163 @@ function summarizeBenchmarkReasoning(runs, fallback) {
 		reasoningSummary: reasoningSummaries.size > 1 ? "mixed" : source.reasoningSummary
 	};
 }
+function benchmarkSecondAttemptValue(attempted, value) {
+	return attempted ? value : null;
+}
+function benchmarkSecondAttemptList(attempted, value) {
+	return attempted ? value : [];
+}
+function buildBenchmarkSecondAttempt(result, input) {
+	return {
+		attempted: input.attempted,
+		reason: input.reason,
+		raw: benchmarkSecondAttemptValue(input.attempted, result.observer.repairedRaw),
+		status: benchmarkSecondAttemptValue(input.attempted, result.repairedClassification?.status ?? null),
+		resultReason: benchmarkSecondAttemptValue(input.attempted, result.repairedClassification?.reason ?? null),
+		pass: benchmarkSecondAttemptValue(input.attempted, result.repairedEvaluation?.pass ?? null),
+		failureReasons: benchmarkSecondAttemptList(input.attempted, result.repairedEvaluation?.failureReasons ?? []),
+		summaries: benchmarkSecondAttemptValue(input.attempted, result.repairedEvaluation?.counts.summaries ?? null),
+		observations: benchmarkSecondAttemptValue(input.attempted, result.repairedEvaluation?.counts.observations ?? null),
+		diagnostics: benchmarkSecondAttemptValue(input.attempted, result.observer.repairedDiagnostics),
+		elapsedMs: benchmarkSecondAttemptValue(input.attempted, result.observer.repairedElapsedMs),
+		usage: benchmarkSecondAttemptValue(input.attempted, result.observer.repairedUsage),
+		quality: benchmarkSecondAttemptValue(input.attempted, input.quality)
+	};
+}
+function estimateBenchmarkAttemptCost(model, usage) {
+	if (model === null) return {
+		total: null,
+		unavailableReason: "model_fallback_unresolved"
+	};
+	const total = estimateExtractionModelCost(model, usage);
+	if (total) return {
+		total,
+		unavailableReason: null
+	};
+	if (usage === null) return {
+		total: null,
+		unavailableReason: "missing_usage"
+	};
+	return {
+		total: null,
+		unavailableReason: "unknown_model_pricing"
+	};
+}
+function rate(numerator, denominator) {
+	return denominator > 0 ? numerator / denominator : null;
+}
+function mean(values) {
+	return values.length > 0 ? values.reduce((sum, value) => sum + value, 0) / values.length : null;
+}
+function countOutputModes(runs, key) {
+	const counts = {};
+	for (const run of runs) counts[run[key]] = (counts[run[key]] ?? 0) + 1;
+	return counts;
+}
+function collectBenchmarkMetricSamples(runs) {
+	return {
+		latencies: runs.flatMap((run) => run.telemetry.totalElapsedMs == null ? [] : [run.telemetry.totalElapsedMs]),
+		usages: runs.flatMap((run) => run.telemetry.totalUsage == null ? [] : [run.telemetry.totalUsage]),
+		summaryScores: runs.flatMap((run) => run.quality == null ? [] : [run.quality.summaryDisposition.score]),
+		structuralScores: runs.flatMap((run) => {
+			if (run.outputValidation === "invalid") return [0];
+			if (run.outputValidation === "valid") return [1];
+			return run.quality == null ? [] : [run.quality.schemaCompliance.score];
+		}),
+		semanticScores: runs.flatMap((run) => run.quality == null ? [] : semanticQualityScore(run.quality))
+	};
+}
+function semanticQualityScore(quality) {
+	if (quality.reviewStatus !== "reviewed") return [];
+	const available = [
+		[quality.summaryDisposition.score, EXTRACTION_BENCHMARK_QUALITY_WEIGHTS.summaryDisposition],
+		[quality.requiredRecall?.score, EXTRACTION_BENCHMARK_QUALITY_WEIGHTS.requiredRecall],
+		[quality.optionalRecall?.score, EXTRACTION_BENCHMARK_QUALITY_WEIGHTS.optionalRecall],
+		[quality.worthinessPrecision?.score, EXTRACTION_BENCHMARK_QUALITY_WEIGHTS.worthinessPrecision],
+		[quality.summaryBreadth?.score, EXTRACTION_BENCHMARK_QUALITY_WEIGHTS.summaryBreadth],
+		[quality.observationSignals?.redundancyAvoidance, EXTRACTION_BENCHMARK_QUALITY_WEIGHTS.redundancyAvoidance],
+		[quality.observationSignals?.segmentation, EXTRACTION_BENCHMARK_QUALITY_WEIGHTS.segmentation]
+	].filter((entry) => entry[0] != null);
+	const weight = available.reduce((sum, [, dimensionWeight]) => sum + dimensionWeight, 0);
+	if (weight === 0) return [];
+	return [available.reduce((sum, [score, dimensionWeight]) => sum + score * dimensionWeight, 0) / weight];
+}
+function benchmarkModeMetrics(runs) {
+	const validOutputs = runs.filter((run) => run.outputValidation === "valid" || run.quality?.schemaCompliance.compliant === true).length;
+	const repairAttempts = runs.filter((run) => run.repairAttempted).length;
+	const retryAttempts = runs.filter((run) => run.retryAttempted).length;
+	const repairOrRetryAttempts = runs.filter((run) => run.repairAttempted || run.retryAttempted).length;
+	const { latencies, usages, summaryScores, structuralScores, semanticScores } = collectBenchmarkMetricSamples(runs);
+	const retainedObservations = runs.reduce((sum, run) => sum + run.observations, 0);
+	const matchingSummaryDispositions = summaryScores.filter((score) => score === 1).length;
+	return {
+		runs: runs.length,
+		validOutputs,
+		validOutputRate: rate(validOutputs, runs.length),
+		repairAttempts,
+		repairRate: rate(repairAttempts, runs.length),
+		retryAttempts,
+		retryRate: rate(retryAttempts, runs.length),
+		repairOrRetryAttempts,
+		repairOrRetryRate: rate(repairOrRetryAttempts, runs.length),
+		knownLatencyRuns: latencies.length,
+		totalLatencyMs: latencies.reduce((sum, value) => sum + value, 0),
+		meanLatencyMs: mean(latencies),
+		knownTokenRuns: usages.length,
+		inputTokens: usages.reduce((sum, usage) => sum + usage.inputTokens, 0),
+		outputTokens: usages.reduce((sum, usage) => sum + usage.outputTokens, 0),
+		totalTokens: usages.reduce((sum, usage) => sum + (usage.totalTokens ?? usage.inputTokens + usage.outputTokens), 0),
+		retainedObservations,
+		meanRetainedObservations: rate(retainedObservations, runs.length),
+		summaryDispositionEvaluated: summaryScores.length,
+		summaryDispositionMatches: matchingSummaryDispositions,
+		summaryDispositionMatchRate: rate(matchingSummaryDispositions, summaryScores.length),
+		structuralQualityEvaluated: structuralScores.length,
+		contractIntegrityRate: mean(structuralScores),
+		semanticQualityEvaluated: semanticScores.length,
+		meanSemanticQuality: mean(semanticScores)
+	};
+}
+function summarizeExtractionBenchmarkReporting(runs) {
+	const byActualOutputMode = {};
+	for (const mode of [
+		"json_schema",
+		"forced_tool",
+		"legacy_xml"
+	]) {
+		const modeRuns = runs.filter((run) => run.actualOutputMode === mode);
+		if (modeRuns.length > 0) byActualOutputMode[mode] = benchmarkModeMetrics(modeRuns);
+	}
+	return {
+		requestedOutputModes: countOutputModes(runs, "requestedOutputMode"),
+		actualOutputModes: countOutputModes(runs, "actualOutputMode"),
+		overall: benchmarkModeMetrics(runs),
+		byActualOutputMode
+	};
+}
+function summarizeExtractionBenchmarkAttempts(attempts, batches) {
+	return {
+		total: attempts.length,
+		shapeQualityTotal: attempts.filter((attempt) => attempt.purpose === "shape_quality").length,
+		shapeQualityPasses: attempts.filter((attempt) => attempt.purpose === "shape_quality" && attempt.status === "pass").length,
+		shapeQualityFails: attempts.filter((attempt) => attempt.purpose === "shape_quality" && attempt.status !== "pass").length,
+		expectedTierTotal: attempts.filter((attempt) => attempt.expectedTier != null).length,
+		expectedTierMatches: attempts.filter((attempt) => attempt.expectedTier != null && attempt.expectedTier === attempt.tier).length,
+		robustnessNoOutput: attempts.filter((attempt) => attempt.purpose === "replay_robustness" && (attempt.status === "observer_no_output" || attempt.status === "output_failure")).length,
+		perBatchStability: batches.map((batch) => {
+			const batchAttempts = attempts.filter((attempt) => attempt.batchId === batch.batchId).toSorted((left, right) => left.iteration - right.iteration);
+			const passes = batchAttempts.filter((attempt) => attempt.status === "pass").length;
+			return {
+				batchId: batch.batchId,
+				purpose: batch.purpose,
+				passes,
+				total: batchAttempts.length,
+				passRate: rate(passes, batchAttempts.length),
+				statuses: batchAttempts.map((attempt) => attempt.status)
+			};
+		})
+	};
+}
 function createMemoryExtractionBenchmarkCommand() {
 	const cmd = new Command("extraction-benchmark").configureHelp(helpStyle).description("Run the formal extraction replay benchmark set and print a cost/quality scoreboard").requiredOption("--benchmark <id>", "benchmark profile id").option("--observer-provider <provider>", "override observer provider for this benchmark run").option("--observer-model <model>", "override observer model for this benchmark run").option("--observer-tier-routing", "use replay-only benchmark-backed observer tier routing").option("--openai-responses", "use OpenAI Responses API for this benchmark run").option("--reasoning-effort <level>", "set OpenAI reasoning.effort for this benchmark run (responses path)").option("--reasoning-summary <mode>", "set OpenAI reasoning.summary for this benchmark run (responses path)").option("--max-output-tokens <n>", "override OpenAI max_output_tokens for this benchmark run (responses path)").option("--observer-temperature <value>", "override observer temperature for this benchmark run").option("--transcript-budget <chars>", "override replay transcript budget in characters for this benchmark run").option("--repetitions <n>", "run every benchmark batch 1-10 times to measure model stability", "1");
 	addDbOption(cmd);
@@ -5255,21 +5412,54 @@ function createMemoryExtractionBenchmarkCommand() {
 				observerExplicitConfigKeys: maxOutputTokens === null ? observerConfig.observerExplicitConfigKeys : [.../* @__PURE__ */ new Set([...observerConfig.observerExplicitConfigKeys ?? [], "observerMaxOutputTokens"])]
 			};
 			const observer = new ObserverClient(observerConfigWithOverrides);
+			const outputFailures = [];
 			const runs = [];
 			for (let iteration = 1; iteration <= repetitions; iteration += 1) for (const batch of benchmark.batches) {
 				const scenarioId = batch.scenarioId ?? benchmark.scenarioId;
-				const result = opts.observerTierRouting === true ? await replayBatchExtractionWithTierRouting(resolveDbOpt(opts), observerConfigWithOverrides, {
-					batchId: batch.batchId,
-					scenarioId,
-					transcriptBudget: transcriptBudget ?? void 0
-				}) : await replayBatchExtraction(resolveDbOpt(opts), observer, {
-					batchId: batch.batchId,
-					scenarioId,
-					transcriptBudget: transcriptBudget ?? void 0
-				});
+				let selectedObserver = extractionReplayObserverIdentity(observer, null);
+				let result;
+				try {
+					result = opts.observerTierRouting === true ? await replayBatchExtractionWithTierRouting(resolveDbOpt(opts), observerConfigWithOverrides, {
+						batchId: batch.batchId,
+						scenarioId,
+						transcriptBudget: transcriptBudget ?? void 0,
+						onOutputFailure: (context) => {
+							selectedObserver = context;
+						}
+					}) : await replayBatchExtraction(resolveDbOpt(opts), observer, {
+						batchId: batch.batchId,
+						scenarioId,
+						transcriptBudget: transcriptBudget ?? void 0
+					});
+				} catch (error) {
+					if (!(error instanceof ObserverOutputError) && !(error instanceof ObserverOutputTransportError)) throw error;
+					if (opts.observerTierRouting !== true) selectedObserver = extractionReplayObserverIdentity(observer, null);
+					const failureCost = estimateBenchmarkAttemptCost(selectedObserver.model, error.telemetry.totalUsage);
+					outputFailures.push({
+						iteration,
+						batchId: batch.batchId,
+						purpose: batch.purpose,
+						status: "output_failure",
+						reason: error instanceof ObserverOutputError ? error.reason : error.code,
+						expectedTier: batch.expectedTier ?? null,
+						...selectedObserver,
+						requestedOutputMode: error.diagnostics.requestedMode,
+						actualOutputMode: error.diagnostics.actualMode,
+						outputValidation: "invalid",
+						repairAttempted: error.diagnostics.repairAttempted,
+						retryAttempted: error.diagnostics.retryAttempted,
+						observations: 0,
+						telemetry: error.telemetry,
+						cost: failureCost,
+						quality: null
+					});
+					continue;
+				}
 				const costModel = result.observer.modelFallbackApplied ? result.observer.resolvedModel : result.observer.resolvedModel ?? result.observer.model;
 				const initialCost = costModel ? estimateExtractionModelCost(costModel, result.observer.initialUsage) : null;
-				const repairCost = costModel ? estimateExtractionModelCost(costModel, result.observer.repairedUsage) : null;
+				const secondAttemptCost = costModel ? estimateExtractionModelCost(costModel, result.observer.repairedUsage) : null;
+				const repairCost = benchmarkSecondAttemptValue(result.observer.repairAttempted, secondAttemptCost);
+				const retryCost = benchmarkSecondAttemptValue(result.observer.retryAttempted, secondAttemptCost);
 				const totalCost = costModel ? estimateExtractionModelCost(costModel, result.observer.totalUsage) : null;
 				const pricing = costModel ? getExtractionModelPricing(costModel) : null;
 				const costUnavailableReason = totalCost ? null : result.observer.modelFallbackApplied && !result.observer.resolvedModel ? "model_fallback_unresolved" : result.observer.totalUsage == null ? "missing_usage" : "unknown_model_pricing";
@@ -5283,14 +5473,14 @@ function createMemoryExtractionBenchmarkCommand() {
 					estimatedCostUsd: initialCost?.totalCostUsd ?? null,
 					expectedSummaryDisposition: batch.expectedSummaryDisposition
 				}) : null;
-				const repairQuality = result.observer.repairedParsed && result.observer.repairedDiagnostics ? scoreExtractionBenchmarkOutput({
+				const secondAttemptQuality = result.observer.repairedParsed && result.observer.repairedDiagnostics ? scoreExtractionBenchmarkOutput({
 					parsed: result.observer.repairedParsed,
 					diagnostics: result.observer.repairedDiagnostics,
 					review: batch.review ?? {
 						status: "unreviewed",
 						reviewerNotes: "No durable-fact review has been recorded for this batch."
 					},
-					estimatedCostUsd: repairCost?.totalCostUsd ?? null,
+					estimatedCostUsd: secondAttemptCost?.totalCostUsd ?? null,
 					expectedSummaryDisposition: batch.expectedSummaryDisposition
 				}) : null;
 				const finalQuality = result.observer.diagnostics ? scoreExtractionBenchmarkOutput({
@@ -5310,6 +5500,16 @@ function createMemoryExtractionBenchmarkCommand() {
 					initialQuality,
 					finalQuality
 				});
+				const repairAttempt = buildBenchmarkSecondAttempt(result, {
+					attempted: result.observer.repairAttempted,
+					reason: null,
+					quality: secondAttemptQuality
+				});
+				const retryAttempt = buildBenchmarkSecondAttempt(result, {
+					attempted: result.observer.retryAttempted,
+					reason: result.observer.retryReason,
+					quality: secondAttemptQuality
+				});
 				runs.push({
 					iteration,
 					batchId: batch.batchId,
@@ -5336,6 +5536,9 @@ function createMemoryExtractionBenchmarkCommand() {
 					resolvedModel: result.observer.resolvedModel,
 					modelFallbackApplied: result.observer.modelFallbackApplied,
 					modelFallbackReason: result.observer.modelFallbackReason,
+					requestedOutputMode: result.observer.requestedOutputMode,
+					actualOutputMode: result.observer.actualOutputMode,
+					outputValidation: result.observer.outputValidation,
 					openaiUseResponses: result.observer.openaiUseResponses,
 					reasoningEffort: result.observer.reasoningEffort,
 					reasoningSummary: result.observer.reasoningSummary,
@@ -5344,6 +5547,9 @@ function createMemoryExtractionBenchmarkCommand() {
 					summaries: result.evaluation.counts.summaries,
 					observations: result.evaluation.counts.observations,
 					repairApplied: result.observer.repairApplied,
+					repairAttempted: result.observer.repairAttempted,
+					retryAttempted: result.observer.retryAttempted,
+					retryReason: result.observer.retryReason,
 					initial: {
 						raw: result.observer.initialRaw,
 						status: result.initialClassification.status,
@@ -5359,18 +5565,19 @@ function createMemoryExtractionBenchmarkCommand() {
 					},
 					repair: {
 						applied: result.observer.repairApplied,
-						raw: result.observer.repairedRaw,
-						status: result.repairedClassification?.status ?? null,
-						reason: result.repairedClassification?.reason ?? null,
-						pass: result.repairedEvaluation?.pass ?? null,
-						failureReasons: result.repairedEvaluation?.failureReasons ?? [],
-						summaries: result.repairedEvaluation?.counts.summaries ?? null,
-						observations: result.repairedEvaluation?.counts.observations ?? null,
-						diagnostics: result.observer.repairedDiagnostics,
-						elapsedMs: result.observer.repairedElapsedMs,
-						usage: result.observer.repairedUsage,
-						quality: repairQuality
+						raw: repairAttempt.raw,
+						status: repairAttempt.status,
+						reason: repairAttempt.resultReason,
+						pass: repairAttempt.pass,
+						failureReasons: repairAttempt.failureReasons,
+						summaries: repairAttempt.summaries,
+						observations: repairAttempt.observations,
+						diagnostics: repairAttempt.diagnostics,
+						elapsedMs: repairAttempt.elapsedMs,
+						usage: repairAttempt.usage,
+						quality: repairAttempt.quality
 					},
+					retry: retryAttempt,
 					telemetry: {
 						totalElapsedMs: result.observer.totalElapsedMs,
 						totalUsage: result.observer.totalUsage
@@ -5379,6 +5586,7 @@ function createMemoryExtractionBenchmarkCommand() {
 					cost: {
 						initial: initialCost,
 						repair: repairCost,
+						retry: retryCost,
 						total: totalCost,
 						unavailableReason: costUnavailableReason
 					},
@@ -5386,68 +5594,55 @@ function createMemoryExtractionBenchmarkCommand() {
 				});
 			}
 			const reviewedQualityRuns = runs.filter((run) => run.quality?.weightedQualityScore != null);
-			const knownCostRuns = runs.filter((run) => run.cost.total != null);
-			const knownElapsedRuns = runs.filter((run) => run.telemetry.totalElapsedMs != null);
+			const attempts = [...runs, ...outputFailures];
+			const knownCostRuns = attempts.filter((run) => run.cost.total != null);
+			const attemptSummary = summarizeExtractionBenchmarkAttempts(attempts, benchmark.batches);
+			const knownElapsedRuns = attempts.filter((run) => run.telemetry.totalElapsedMs != null);
 			const summary = {
 				repetitions,
-				total: runs.length,
-				shapeQualityTotal: runs.filter((run) => run.purpose === "shape_quality").length,
-				shapeQualityPasses: runs.filter((run) => run.purpose === "shape_quality" && run.status === "pass").length,
-				shapeQualityFails: runs.filter((run) => run.purpose === "shape_quality" && run.status === "shape_fail").length,
-				expectedTierTotal: runs.filter((run) => run.expectedTier != null).length,
-				expectedTierMatches: runs.filter((run) => run.expectedTier != null && run.expectedTier === run.tier).length,
-				robustnessNoOutput: runs.filter((run) => run.status === "observer_no_output").length,
+				...attemptSummary,
 				summaryDispositionTotal: runs.filter((run) => run.quality != null).length,
 				summaryDispositionMatches: runs.filter((run) => run.quality?.summaryDisposition.score === 1).length,
 				reviewedQualityRuns: reviewedQualityRuns.length,
 				knownCostRuns: knownCostRuns.length,
-				unknownCostRuns: runs.length - knownCostRuns.length,
-				missingUsageRuns: runs.filter((run) => run.cost.unavailableReason === "missing_usage").length,
-				unknownPricingRuns: runs.filter((run) => run.cost.unavailableReason === "unknown_model_pricing").length,
-				fallbackUnresolvedRuns: runs.filter((run) => run.cost.unavailableReason === "model_fallback_unresolved").length,
+				unknownCostRuns: attempts.length - knownCostRuns.length,
+				missingUsageRuns: attempts.filter((run) => run.cost.unavailableReason === "missing_usage").length,
+				unknownPricingRuns: attempts.filter((run) => run.cost.unavailableReason === "unknown_model_pricing").length,
+				fallbackUnresolvedRuns: attempts.filter((run) => run.cost.unavailableReason === "model_fallback_unresolved").length,
 				totalKnownCostUsd: knownCostRuns.reduce((sum, run) => sum + (run.cost.total?.totalCostUsd ?? 0), 0),
 				knownElapsedRuns: knownElapsedRuns.length,
 				totalKnownElapsedMs: knownElapsedRuns.reduce((sum, run) => sum + (run.telemetry.totalElapsedMs ?? 0), 0),
-				perBatchStability: benchmark.batches.map((batch) => {
-					const batchRuns = runs.filter((run) => run.batchId === batch.batchId);
-					const passes = batchRuns.filter((run) => run.status === "pass").length;
-					return {
-						batchId: batch.batchId,
-						purpose: batch.purpose,
-						passes,
-						total: batchRuns.length,
-						passRate: batchRuns.length > 0 ? passes / batchRuns.length : null,
-						statuses: batchRuns.map((run) => run.status)
-					};
-				})
+				output: summarizeExtractionBenchmarkReporting(attempts),
+				outputFailures
 			};
-			const uniqueObserverKeys = Array.from(new Set(runs.map((run) => `${run.provider}::${run.model}::${run.transport}`)));
-			const benchmarkReasoning = summarizeBenchmarkReasoning(runs, {
+			const uniqueObserverKeys = Array.from(new Set(attempts.map((run) => `${run.provider}::${run.model}::${run.transport}`)));
+			const firstObserver = attempts[0];
+			const benchmarkReasoning = summarizeBenchmarkReasoning(attempts, {
 				reasoningEffort: observer.reasoningEffort,
 				reasoningSummary: observer.reasoningSummary
 			});
 			const observerSummary = opts.observerTierRouting === true ? {
-				provider: uniqueObserverKeys.length === 1 ? runs[0]?.provider ?? observer.provider : "mixed",
-				model: uniqueObserverKeys.length === 1 ? runs[0]?.model ?? observer.model : "mixed",
-				transport: uniqueObserverKeys.length === 1 ? runs[0]?.transport ?? "unknown" : "mixed",
+				provider: uniqueObserverKeys.length === 1 ? firstObserver?.provider ?? observer.provider : "mixed",
+				model: uniqueObserverKeys.length === 1 ? firstObserver?.model ?? observer.model : "mixed",
+				transport: uniqueObserverKeys.length === 1 ? firstObserver?.transport ?? "unknown" : "mixed",
 				tierRouting: true,
-				openaiUseResponses: uniqueObserverKeys.length === 1 ? runs[0]?.openaiUseResponses ?? observer.openaiUseResponses : null,
+				openaiUseResponses: uniqueObserverKeys.length === 1 ? firstObserver?.openaiUseResponses ?? observer.openaiUseResponses : null,
 				reasoningEffort: benchmarkReasoning.reasoningEffort,
 				reasoningSummary: benchmarkReasoning.reasoningSummary,
-				maxOutputTokens: uniqueObserverKeys.length === 1 ? runs[0]?.maxOutputTokens ?? null : null,
-				temperature: uniqueObserverKeys.length === 1 ? runs[0]?.temperature ?? null : null,
+				maxOutputTokens: uniqueObserverKeys.length === 1 ? firstObserver?.maxOutputTokens ?? null : null,
+				temperature: uniqueObserverKeys.length === 1 ? firstObserver?.temperature ?? null : null,
 				transcriptBudget: transcriptBudget ?? null,
 				selectedObservers: uniqueObserverKeys
 			} : {
 				provider: observer.provider,
 				model: observer.model,
-				transport: runs[0]?.transport ?? observer.getStatus().runtime,
+				transport: firstObserver?.transport ?? observer.getStatus().runtime,
 				tierRouting: false,
 				openaiUseResponses: observer.openaiUseResponses,
 				reasoningEffort: benchmarkReasoning.reasoningEffort,
 				reasoningSummary: benchmarkReasoning.reasoningSummary,
-				maxOutputTokens: runs[0]?.maxOutputTokens ?? null,
-				temperature: runs[0]?.temperature ?? null,
+				maxOutputTokens: firstObserver?.maxOutputTokens ?? null,
+				temperature: firstObserver?.temperature ?? null,
 				transcriptBudget: transcriptBudget ?? null,
 				selectedObservers: uniqueObserverKeys
 			};
@@ -5486,14 +5681,23 @@ function createMemoryExtractionBenchmarkCommand() {
 				`Summary disposition matches: ${summary.summaryDispositionMatches}/${summary.summaryDispositionTotal}`,
 				`Reviewed quality runs: ${summary.reviewedQualityRuns} (compare per-run dimensions; scores are fixture-specific)`,
 				`Known estimated cost: $${summary.totalKnownCostUsd.toFixed(6)} (${summary.knownCostRuns}/${summary.total}; missing usage=${summary.missingUsageRuns}, unknown pricing=${summary.unknownPricingRuns}, unresolved fallback=${summary.fallbackUnresolvedRuns})`,
-				`Known elapsed time: ${summary.totalKnownElapsedMs}ms (${summary.knownElapsedRuns}/${summary.total} run(s))`
+				`Known elapsed time: ${summary.totalKnownElapsedMs}ms (${summary.knownElapsedRuns}/${summary.total} run(s))`,
+				`Output modes requested/actual: ${JSON.stringify(summary.output.requestedOutputModes)} / ${JSON.stringify(summary.output.actualOutputModes)}`,
+				`Valid output: ${summary.output.overall.validOutputs}/${summary.output.overall.runs}; repair/retry: ${summary.output.overall.repairAttempts}/${summary.output.overall.retryAttempts}; mean latency: ${summary.output.overall.meanLatencyMs ?? "n/a"}ms`,
+				`Tokens input/output: ${summary.output.overall.inputTokens}/${summary.output.overall.outputTokens} (${summary.output.overall.knownTokenRuns} measured run(s)); retained observations: ${summary.output.overall.retainedObservations}`,
+				`Summary disposition matches: ${summary.output.overall.summaryDispositionMatches}/${summary.output.overall.summaryDispositionEvaluated}; contract integrity/semantic quality: ${summary.output.overall.contractIntegrityRate?.toFixed(3) ?? "n/a"}/${summary.output.overall.meanSemanticQuality?.toFixed(3) ?? "n/a"}`
 			].join("\n"));
 			for (const run of runs) {
 				const qualityLabel = run.quality?.weightedQualityScore == null ? "n/a" : run.quality.weightedQualityScore.toFixed(3);
 				const costLabel = run.cost.total == null ? "n/a" : `$${run.cost.total.totalCostUsd.toFixed(6)}`;
 				const latencyLabel = run.telemetry.totalElapsedMs == null ? "n/a" : `${run.telemetry.totalElapsedMs}ms`;
 				const missingRequired = run.quality?.requiredRecall.missingLabelIds.join(",") || "none";
-				p.log.message(`  [${run.batchId}#${run.iteration}] ${run.status.padEnd(18)} ${run.complexity.padEnd(10)} tier=${run.tier.padEnd(6)} expected=${(run.expectedTier ?? "n/a").padEnd(6)} disposition=${run.quality?.summaryDisposition.actual ?? "n/a"}/${run.expectedSummaryDisposition} span=${String(run.analysis.eventSpan).padEnd(3)} prompts=${run.analysis.promptCount} tools=${String(run.analysis.toolCount).padEnd(2)} transcript=${run.analysis.transcriptLength} ${run.provider}/${run.model} [${run.transport}] initial=${run.initial.summaries}s/${run.initial.observations}o final=${run.summaries}s/${run.observations}o quality=${qualityLabel} coverage=${run.quality?.weightedQualityCoverage?.toFixed(3) ?? "n/a"} required_missing=${missingRequired} cost=${costLabel} latency=${latencyLabel} schema_loss=${run.initial.diagnostics?.dataLoss === true ? "yes" : "no"} fallback=${run.modelFallbackApplied ? "yes" : "no"} repair=${run.repairApplied ? "yes" : "no"} — ${run.label}`);
+				p.log.message(`  [${run.batchId}#${run.iteration}] ${run.status.padEnd(18)} ${run.complexity.padEnd(10)} tier=${run.tier.padEnd(6)} expected=${(run.expectedTier ?? "n/a").padEnd(6)} mode=${run.requestedOutputMode}/${run.actualOutputMode} valid=${run.outputValidation} disposition=${run.quality?.summaryDisposition.actual ?? "n/a"}/${run.expectedSummaryDisposition} span=${String(run.analysis.eventSpan).padEnd(3)} prompts=${run.analysis.promptCount} tools=${String(run.analysis.toolCount).padEnd(2)} transcript=${run.analysis.transcriptLength} ${run.provider}/${run.model} [${run.transport}] initial=${run.initial.summaries}s/${run.initial.observations}o final=${run.summaries}s/${run.observations}o quality=${qualityLabel} coverage=${run.quality?.weightedQualityCoverage?.toFixed(3) ?? "n/a"} required_missing=${missingRequired} cost=${costLabel} latency=${latencyLabel} schema_loss=${run.initial.diagnostics?.dataLoss === true ? "yes" : "no"} fallback=${run.modelFallbackApplied ? "yes" : "no"} repair=${run.repairApplied ? "yes" : "no"} retry=${run.retryAttempted ? run.retryReason ?? "yes" : "no"} — ${run.label}`);
+			}
+			for (const failure of outputFailures.toSorted((left, right) => left.iteration - right.iteration || left.batchId - right.batchId)) {
+				const costLabel = failure.cost.total == null ? "n/a" : `$${failure.cost.total.totalCostUsd.toFixed(6)}`;
+				const latencyLabel = failure.telemetry.totalElapsedMs == null ? "n/a" : `${failure.telemetry.totalElapsedMs}ms`;
+				p.log.error(`  [${failure.batchId}#${failure.iteration}] output_failure tier=${failure.tier ?? "n/a"} expected=${failure.expectedTier ?? "n/a"} mode=${failure.requestedOutputMode}/${failure.actualOutputMode} model=${failure.model ?? "unresolved"} cost=${costLabel} latency=${latencyLabel} — ${failure.reason}`);
 			}
 			p.outro("done");
 		} catch (error) {
@@ -5609,6 +5813,8 @@ var FORBIDDEN_INTERNAL_KEYS = /* @__PURE__ */ new Set([
 	"title"
 ]);
 var INTERNAL_LEDGER_KEYS = /* @__PURE__ */ new Set([
+	"automatic_recall",
+	"evaluation_key",
 	"action",
 	"attempt_id",
 	"started_at",
@@ -5626,7 +5832,8 @@ var INTERNAL_LEDGER_KEYS = /* @__PURE__ */ new Set([
 var INTERNAL_LEDGER_ACTIONS = /* @__PURE__ */ new Set([
 	"record",
 	"delivery",
-	"cache_reuse"
+	"cache_reuse",
+	"recall"
 ]);
 var INTERNAL_RETRIEVAL_STATUSES = /* @__PURE__ */ new Set(["skipped", "failed"]);
 var INTERNAL_DELIVERY_STATUSES = /* @__PURE__ */ new Set([
@@ -5785,12 +5992,28 @@ async function withStore(opts, errorCode, run) {
 		store?.close();
 	}
 }
+function ledgerAutomaticContext(payload) {
+	if (payload?.source?.trim().toLowerCase() !== "opencode") return null;
+	if (!payload.source_session_id) return null;
+	return {
+		source: "opencode",
+		hostSessionId: payload.source_session_id
+	};
+}
+async function readOptionalLedgerPayload() {
+	try {
+		return await readInternalLedgerPayload();
+	} catch {
+		return;
+	}
+}
 async function packAction(context, opts) {
 	await withStore(opts, "pack_failed", async (store) => {
 		const { limit, budget, filters, renderOptions } = buildPackRequestOptions(opts, { envProject: process.env.CODEMEM_PROJECT });
 		let result;
 		if (opts.internalLedger) {
-			const artifacts = await store.buildMemoryPackWithTraceAsync(context, limit, budget, filters, renderOptions);
+			const ledgerPayload = await readOptionalLedgerPayload();
+			const artifacts = await store.buildMemoryPackWithTraceAsync(context, limit, budget, filters, renderOptions, ledgerAutomaticContext(ledgerPayload));
 			result = artifacts.response;
 			let artifactFingerprint;
 			try {
@@ -5798,12 +6021,13 @@ async function packAction(context, opts) {
 			} catch {}
 			let ledgerOutcome;
 			try {
-				const ledgerPayload = await readInternalLedgerPayload();
+				if (!ledgerPayload) throw new Error("internal ledger metadata unavailable");
 				ledgerOutcome = handleInstrumentedPackLedger(store.db, ledgerPayload, context, filters, artifacts);
 			} catch {}
 			emitPackResult(context, opts, result, artifactFingerprint, ledgerOutcome?.ok === false && ledgerOutcome.reason === "idempotency_conflict" ? ledgerOutcome : void 0);
 			return;
-		} else result = await store.buildMemoryPackAsync(context, limit, budget, filters, renderOptions);
+		}
+		result = await store.buildMemoryPackAsync(context, limit, budget, filters, renderOptions);
 		emitPackResult(context, opts, result);
 	});
 }
@@ -5850,7 +6074,30 @@ addJsonOption(traceCmd);
 traceCmd.action(traceAction);
 packCmd.addCommand(traceCmd);
 var packCommand = packCmd;
+function handleRecallMeasurement(db, payload) {
+	if (payload.automatic_recall === void 0 && payload.action !== "recall") return null;
+	const isValidMeasurement = isAutomaticRecallMeasurement(payload.automatic_recall) && typeof payload.evaluation_key === "string" && /^[a-f0-9]{64}$/.test(payload.evaluation_key);
+	if (payload.action === "delivery") {
+		if (!isValidMeasurement) return null;
+		try {
+			recordAutomaticRecall(db, payload.attempt_id, payload.evaluation_key, payload.automatic_recall);
+		} catch {}
+		return null;
+	}
+	if (payload.action !== "recall" || !isValidMeasurement) throw new PackUsageError("invalid automatic recall measurement");
+	const outcome = recordAutomaticRecall(db, payload.attempt_id, payload.evaluation_key, payload.automatic_recall);
+	if (!outcome.ok) throw new PackUsageError(outcome.reason);
+	return outcome.value;
+}
 function handlePromptPackLedger(db, payload) {
+	if (payload.action === "delivery") {
+		const outcome = handlePromptPackLifecycle(db, payload);
+		handleRecallMeasurement(db, payload);
+		return outcome;
+	}
+	return handleRecallMeasurement(db, payload) ?? handlePromptPackLifecycle(db, payload);
+}
+function handlePromptPackLifecycle(db, payload) {
 	const metadata = attemptMetadata(payload);
 	if (payload.action === "delivery") {
 		const status = payload.delivery_status;