ds4-context-engine 0.1.1 → 0.2.0-beta.1

package/src/extension/runtime.ts

@@ -1,6 +1,7 @@
 import { randomUUID } from "node:crypto";
 import { existsSync, realpathSync } from "node:fs";
-import { join, resolve } from "node:path";
+import { homedir } from "node:os";
+import { join, parse, resolve } from "node:path";
 import type {
   Api,
   AssistantMessage,
@@ -14,6 +15,7 @@ import {
   type SessionBeforeCompactEvent,
   type SessionCompactEvent,
 } from "@earendil-works/pi-coding-agent";
+import type { RuntimeCapabilityNegotiation } from "ds4-context-core/adapter/runtime-adapter";
 import {
   ArtifactManager,
   disabledArtifactDiagnostics,
@@ -41,7 +43,12 @@ export type {
   CompactionPhase,
   SummaryGraphDiagnostics,
 } from "../pi-adapter/compaction-coordinator.ts";
-import { loadConfig, resolveDatabasePath, type LoadedConfig } from "ds4-context-core/config/config-loader";
+import {
+  loadConfig,
+  resolveDatabasePath,
+  resolveRankingModelPath,
+  type LoadedConfig,
+} from "ds4-context-core/config/config-loader";
 import { createDefaultConfig, type Ds4ContextConfig } from "ds4-context-core/config/config";
 import { calculateContextBudget, type ContextBudget } from "ds4-context-core/core/budget-manager";
 import {
@@ -61,6 +68,7 @@ import type {
 } from "ds4-context-core/manifest/context-manifest";
 import type { ExcludedContextSource, ObservedTool } from "ds4-context-core/manifest/observer";
 import {
+  disabledCrossSessionMemoryDiagnostics,
   disabledMemoryDiagnostics,
   MemoryManager,
   type MemoryDiagnostics,
@@ -69,14 +77,36 @@ import {
 import {
   MEMORY_CUSTOM_ENTRY_TYPE,
   PIN_CUSTOM_ENTRY_TYPE,
+  type CrossSessionMemoryDiagnostics,
   type MemoryItem,
   type MemoryMutation,
   type MemoryScope,
   type PinItem,
   type PinMutation,
   type PinScope,
+  type ProjectMemorySource,
 } from "ds4-context-core/memory/memory-types";
 import { planManagedContext, type SupplementalContextMessage } from "ds4-context-core/planner/context-planner";
+import {
+  disabledContextQualityDiagnostics,
+  evaluateManifestQuality,
+  type ContextQualityDiagnostics,
+} from "ds4-context-core/quality/context-quality";
+import {
+  createRankingFeatures,
+  createRankingFeedback,
+  disabledRankingDiagnostics,
+  loadLearnedRankingModel,
+  rankCandidates,
+  RANKING_FEEDBACK_CUSTOM_ENTRY_TYPE,
+  saveLearnedRankingModel,
+  trainLearnedRankingModel,
+  type LearnedRankingModel,
+  type RankingDiagnostics,
+  type RankingFeatureVector,
+  type RankingFeedbackEntry,
+  type RankingFeedbackLabel,
+} from "ds4-context-core/ranking/learned-ranker";
 import {
   disabledPrivacyDiagnostics,
   emptyPrivacyCounts,
@@ -89,12 +119,15 @@ import {
   ProjectKnowledgeManager,
   type ProjectKnowledgeDiagnostics,
 } from "ds4-context-core/project/project-knowledge";
+import type { EmbeddingPort } from "ds4-context-core/retrieval/embedding";
+import { SemanticEmbeddingIndex } from "ds4-context-core/retrieval/semantic-index";
 import {
   emptyRetrievalDiagnostics,
   HistoricalRetrievalEngine,
   type RetrievalDiagnostics,
 } from "ds4-context-core/retrieval/retrieval-engine";
 import { currentRequestText } from "ds4-context-core/retrieval/task-descriptor";
+import { LocalFeatureHashEmbedding } from "../pi-adapter/local-embedding.ts";
 import { ContextDatabase, type DatabaseHealth, type SessionIndexStats } from "ds4-context-core/persistence/sqlite";
 import {
   activeTools,
@@ -103,6 +136,9 @@ import {
   findPiPinnedMessageIndices,
 } from "../pi-adapter/context-observer.ts";
 import { projectSessionFileMutations } from "../pi-adapter/memory-adapter.ts";
+import { ProjectMemorySynchronizer } from "../pi-adapter/project-memory-sync.ts";
+import { projectRankingLabels } from "../pi-adapter/ranking-adapter.ts";
+import { negotiatePiRuntimeCapabilities } from "../pi-adapter/runtime-contract.ts";
 import { PiSessionIndexer, type SessionIndexResult } from "../pi-adapter/session-indexer.ts";
 import { snapshotModel, snapshotSession, type PiSessionSnapshot } from "../pi-adapter/session-reader.ts";
 import { silentLogger, StructuredLogger, type Logger } from "ds4-context-core/shared/logging";
@@ -117,6 +153,24 @@ import {
 export type RuntimePhase = "idle" | "initializing" | "disabled" | "observer" | "managed" | "degraded" | "closed";
 
 const READ_ONLY_PROJECT_TOOLS = new Set(["read", "grep", "find", "ls"]);
+const MAX_PRIOR_RANKING_OUTCOMES = 50_000;
+
+function comparablePath(path: string): string {
+  let canonical: string;
+  try {
+    canonical = realpathSync(path);
+  } catch {
+    canonical = resolve(path);
+  }
+  return process.platform === "win32" ? canonical.toLowerCase() : canonical;
+}
+
+export function isBroadProjectRoot(projectPath: string, homePath = homedir()): boolean {
+  const project = comparablePath(projectPath);
+  const home = comparablePath(homePath);
+  const filesystemRoot = comparablePath(parse(project).root);
+  return project === home || project === filesystemRoot;
+}
 
 export interface RuntimeDependencies {
   agentDir: string;
@@ -125,6 +179,8 @@ export interface RuntimeDependencies {
   now?: () => number;
   idGenerator?: () => string;
   logSink?: (line: string) => void;
+  /** Optional runtime-owned embedding provider; required for configured remote models. */
+  embeddingPort?: EmbeddingPort;
 }
 
 interface PreparedPrivacyContext {
@@ -231,7 +287,30 @@ export interface CreateMemoryInput {
   sourceEntryIds: string[];
 }
 
-export type CustomEntryAppender = (customType: string, data: MemoryMutation | PinMutation) => void;
+export type CustomEntryAppender = (
+  customType: string,
+  data: MemoryMutation | PinMutation | RankingFeedbackEntry,
+) => void;
+
+export interface RuntimeRankingDiagnostics extends RankingDiagnostics {
+  modelPath?: string;
+  modelLoaded: boolean;
+  promoted: boolean;
+  trainingSamples: number;
+  malformedFeedback: number;
+  duplicateFeedback: number;
+  warnings: string[];
+}
+
+export interface RankingTrainingResult {
+  modelId: string;
+  modelPath: string;
+  sampleCount: number;
+  positiveSamples: number;
+  negativeSamples: number;
+  repositoryCount: number;
+  warnings: string[];
+}
 
 export interface RuntimeDiagnostics {
   extensionVersion: string;
@@ -240,6 +319,7 @@ export interface RuntimeDiagnostics {
   phase: RuntimePhase;
   enabled: boolean;
   contextMode: "observer" | "managed";
+  adapter: RuntimeCapabilityNegotiation;
   session?: PiSessionSnapshot;
   model?: { provider: string; id: string };
   databasePath?: string;
@@ -253,6 +333,8 @@ export interface RuntimeDiagnostics {
   privacy: PrivacyDiagnostics;
   modelAwareness?: ModelAwarenessManifest;
   nativeContinuation: NativeContinuationDiagnostics;
+  quality: ContextQualityDiagnostics;
+  ranking: RuntimeRankingDiagnostics;
   artifacts: ArtifactDiagnostics;
   compaction: CompactionDiagnostics;
   lastIndexResult?: SessionIndexResult;
@@ -275,10 +357,15 @@ export class Ds4ContextRuntime {
   private pendingManifestId?: string;
   private retrievalEngine?: HistoricalRetrievalEngine;
   private lastRetrieval: RetrievalDiagnostics = emptyRetrievalDiagnostics();
+  private semanticIndex?: SemanticEmbeddingIndex;
+  private semanticFallbackReason?: string;
   private projectKnowledge?: ProjectKnowledgeManager;
   private lastProject: ProjectKnowledgeDiagnostics = emptyProjectDiagnostics();
   private projectRefreshPending = false;
+  private readonly projectLeaseOwnerId = randomUUID();
   private memoryManager?: MemoryManager;
+  private projectMemorySynchronizer?: ProjectMemorySynchronizer;
+  private lastCrossSessionMemory: CrossSessionMemoryDiagnostics = disabledCrossSessionMemoryDiagnostics();
   private lastMemory: MemoryDiagnostics = disabledMemoryDiagnostics();
   private privacyEngine?: PrivacyPolicyEngine;
   private lastPrivacy: PrivacyDiagnostics = disabledPrivacyDiagnostics();
@@ -294,6 +381,18 @@ export class Ds4ContextRuntime {
   private compaction?: CompactionCoordinator;
   private lastIndexResult?: SessionIndexResult;
   private lastIndexError?: string;
+  private lastQualityError?: string;
+  private rankingModel?: LearnedRankingModel;
+  private rankingModelPath?: string;
+  private lastRanking: RankingDiagnostics = disabledRankingDiagnostics();
+  private readonly rankingCandidates = new Map<string, RankingFeatureVector>();
+  private readonly priorRankingSelection = new Map<string, number>();
+  private rankingWarnings: string[] = [];
+  private readonly pendingQuality: Array<{
+    manifest: ContextManifest;
+    contextConfig: Ds4ContextConfig["context"];
+    recordedAt: number;
+  }> = [];
   private lastError?: string;
   private logger: Logger = silentLogger;
   private readonly now: () => number;
@@ -309,14 +408,25 @@ export class Ds4ContextRuntime {
   }
 
   openSession(ctx: ExtensionContext): void {
+    this.flushContextQuality();
     this.closeDatabase();
     this.phase = "initializing";
     this.lastError = undefined;
     this.lastIndexError = undefined;
+    this.lastQualityError = undefined;
+    this.rankingModel = undefined;
+    this.rankingModelPath = undefined;
+    this.lastRanking = disabledRankingDiagnostics(this.config.ranking.mode);
+    this.rankingCandidates.clear();
+    this.priorRankingSelection.clear();
+    this.rankingWarnings = [];
+    this.pendingQuality.length = 0;
     this.lastIndexResult = undefined;
     this.lastManifest = undefined;
     this.pendingManifestId = undefined;
     this.retrievalEngine = undefined;
+    this.semanticIndex = undefined;
+    this.semanticFallbackReason = undefined;
     this.lastRetrieval = emptyRetrievalDiagnostics(
       this.config.context.maxRetrievedHistoryTokens,
       this.config.retrieval.maxResults,
@@ -325,6 +435,8 @@ export class Ds4ContextRuntime {
     this.lastProject = emptyProjectDiagnostics();
     this.projectRefreshPending = false;
     this.memoryManager = undefined;
+    this.projectMemorySynchronizer = undefined;
+    this.lastCrossSessionMemory = disabledCrossSessionMemoryDiagnostics();
     this.lastMemory = disabledMemoryDiagnostics();
     this.privacyEngine = undefined;
     this.lastPrivacy = disabledPrivacyDiagnostics();
@@ -339,7 +451,7 @@ export class Ds4ContextRuntime {
     this.lastArtifacts = disabledArtifactDiagnostics();
     this.compaction = undefined;
     this.observation = undefined;
-    this.session = snapshotSession(ctx);
+    this.session = this.snapshotCanonicalSession(ctx);
 
     try {
       this.loadedConfig = loadConfig({
@@ -383,13 +495,22 @@ export class Ds4ContextRuntime {
       this.database = ContextDatabase.open(this.databasePath, {
         logger: this.logger,
         now: this.now(),
+        busyTimeoutMs: this.config.storage.busyTimeoutMs,
+        writeRetryTimeoutMs: this.config.storage.writeRetryTimeoutMs,
       });
+      this.initializeRanking();
 
       this.indexer = new PiSessionIndexer(this.database.sessionIndex, {
         logger: this.logger,
         now: this.now,
       });
-      this.retrievalEngine = new HistoricalRetrievalEngine(this.database.sessionIndex);
+      this.initializeSemanticRetrieval();
+      this.retrievalEngine = new HistoricalRetrievalEngine(
+        this.database.sessionIndex,
+        undefined,
+        this.semanticIndex,
+        this.semanticFallbackReason,
+      );
       this.lastRetrieval = emptyRetrievalDiagnostics(
         this.config.context.maxRetrievedHistoryTokens,
         this.config.retrieval.maxResults,
@@ -674,6 +795,7 @@ export class Ds4ContextRuntime {
       const preparedPrivacy = this.preparePrivacyContext(event, ctx, pi);
       if (preparedPrivacy.changed) privacyFallback = { messages: preparedPrivacy.event.messages };
       this.syncSessionIndex(ctx);
+      this.refreshProjectMemorySources();
       this.refreshProjectIndexIfPending();
       if (this.config.context.mode !== "managed") {
         this.lastRetrieval = emptyRetrievalDiagnostics(
@@ -765,6 +887,7 @@ export class Ds4ContextRuntime {
       let result: { messages?: ContextEvent["messages"] } | undefined = privacyFallback;
       if (this.config.context.mode === "managed" && budget) {
         const planningStartedAt = this.now();
+        this.rankingCandidates.clear();
         const requestText = currentRequestText(effectiveEvent.messages);
         const activeEntryIds = new Set(ctx.sessionManager.getBranch().map((entry) => entry.id));
         const memorySelection: MemorySelection = this.memoryManager?.select(requestText, activeEntryIds) ?? {
@@ -814,6 +937,10 @@ export class Ds4ContextRuntime {
             sourceIds: [evidence.item.id],
             score: 90 + Math.min(0.999999, Math.max(0, evidence.score) / 1_000),
             reason: evidence.reason,
+            rankingFeatures: createRankingFeatures({
+              ...evidence.rankingFeatures,
+              priorSelected: this.priorRankingSelection.get(`memory:${evidence.item.id}`) ?? 0,
+            }),
             ...(evidence.item.classification ? { classification: evidence.item.classification } : {}),
           })),
           ...retrieval.selected.map((evidence) => ({
@@ -823,6 +950,10 @@ export class Ds4ContextRuntime {
             sourceIds: [evidence.entryId],
             score: 85 + Math.min(0.999999, Math.max(0, evidence.score) / 1_000),
             reason: `Historical evidence: ${evidence.reason}`,
+            rankingFeatures: createRankingFeatures({
+              ...evidence.rankingFeatures,
+              priorSelected: this.priorRankingSelection.get(`retrieval:${evidence.entryId}`) ?? 0,
+            }),
           })),
           ...project.selected.map((evidence) => ({
             id: `project:${evidence.snippetId}`,
@@ -832,6 +963,10 @@ export class Ds4ContextRuntime {
             score: 80 + Math.min(0.999999, Math.max(0, evidence.score) / 1_000),
             reason: `Project source: ${evidence.reason}`,
             projectSnippet: evidence.manifestRef,
+            rankingFeatures: createRankingFeatures({
+              ...evidence.rankingFeatures,
+              priorSelected: this.priorRankingSelection.get(`project:${evidence.snippetId}`) ?? 0,
+            }),
           })),
         ];
         const privacyExcludedSources: ExcludedContextSource[] = [];
@@ -870,13 +1005,45 @@ export class Ds4ContextRuntime {
           ...this.lastPrivacy,
           excludedSources: this.lastPrivacy.excludedSources + privacyExcludedSources.length,
         };
+        const learnedCandidates = supplementalMessages.flatMap((supplement) =>
+          supplement.kind !== "pin" && supplement.rankingFeatures
+            ? [{
+                id: supplement.id,
+                staticScore: supplement.score,
+                features: supplement.rankingFeatures,
+                value: supplement,
+              }]
+            : []
+        );
+        const ranking = rankCandidates(learnedCandidates, {
+          mode: this.config.ranking.mode,
+          ...(this.rankingModel ? { model: this.rankingModel } : {}),
+          now: this.now,
+          maxLatencyMs: this.config.ranking.maxLatencyMs,
+        });
+        this.lastRanking = ranking.diagnostics;
+        const rankedById = new Map(ranking.ranked.map((candidate) => [candidate.id, candidate]));
+        const rankedSupplementalMessages = supplementalMessages.map((supplement) => {
+          const ranked = rankedById.get(supplement.id);
+          if (!ranked) return supplement;
+          return {
+            ...supplement,
+            score: ranked.effectiveScore,
+            reason: ranking.diagnostics.status === "active"
+              ? `${supplement.reason}; learned rank ${ranked.learnedScore.toFixed(6)}`
+              : supplement.reason,
+          };
+        });
         const plan = planManagedContext({
           messages: effectiveEvent.messages,
           fixedTokens: baseline.composition.systemTokens + baseline.composition.toolTokens,
           budget,
           config: effectiveContextConfig,
           pinnedMessageIndices: findPiPinnedMessageIndices(effectiveEvent, ctx),
-          supplementalMessages,
+          supplementalMessages: rankedSupplementalMessages,
+          ...(ranking.diagnostics.status === "active"
+            ? { supplementalSelectionOrder: ranking.ranked.map((candidate) => candidate.id) }
+            : {}),
           ...(this.config.privacy.enabled
             ? {
                 messageClassifications: preparedPrivacy.messageClassifications,
@@ -884,6 +1051,11 @@ export class Ds4ContextRuntime {
               }
             : {}),
         });
+        if (plan.mode === "managed") {
+          for (const candidate of learnedCandidates) {
+            this.rankingCandidates.set(candidate.id, { ...candidate.features });
+          }
+        }
         const selectedPinIds = new Set(
           plan.selected.flatMap((metadata) => metadata.kind === "pin" && metadata.sourceId ? [metadata.sourceId] : []),
         );
@@ -912,6 +1084,16 @@ export class Ds4ContextRuntime {
           plan.selected.flatMap((metadata) => metadata.projectSnippet?.snippetId ? [metadata.projectSnippet.snippetId] : []),
         );
         const plannerSelectedProject = project.selected.filter((evidence) => selectedProjectIds.has(evidence.snippetId));
+        for (const supplement of rankedSupplementalMessages) {
+          if (supplement.kind === "pin" || !supplement.rankingFeatures) continue;
+          const selected = supplement.kind === "memory"
+            ? selectedMemoryIds.has(supplement.sourceIds[0] ?? "")
+            : supplement.kind === "retrieval"
+              ? selectedRetrievalIds.has(supplement.sourceIds[0] ?? "")
+              : Boolean(supplement.projectSnippet?.snippetId
+                && selectedProjectIds.has(supplement.projectSnippet.snippetId));
+          this.rememberPriorRankingSelection(supplement.id, selected ? 1 : 0);
+        }
         this.lastProject = {
           ...project,
           plannerExcludedCount: project.selected.length - plannerSelectedProject.length,
@@ -986,6 +1168,7 @@ export class Ds4ContextRuntime {
 
       manifest = {
         ...manifest,
+        ranking: { ...this.lastRanking },
         nativeContinuation: this.nativeContinuation.initialManifest(
           manifest.provider,
           manifest.model,
@@ -1011,6 +1194,11 @@ export class Ds4ContextRuntime {
         observedAt,
         ...(budget ? { budget } : {}),
       };
+      if (this.config.quality.enabled) {
+        this.pendingQuality.push({ manifest, contextConfig: effectiveContextConfig, recordedAt: observedAt });
+        const pendingLimit = Math.min(this.config.quality.maxSamples, 256);
+        if (this.pendingQuality.length > pendingLimit) this.pendingQuality.splice(0, this.pendingQuality.length - pendingLimit);
+      }
       this.lastManifest = manifest;
       this.pendingManifestId = manifest.id;
       if (this.config.diagnostics.storeContextManifest && this.session?.sessionFile) {
@@ -1315,8 +1503,10 @@ export class Ds4ContextRuntime {
   }
 
   afterAgentSettled(ctx: ExtensionContext): void {
+    this.flushContextQuality();
     if (this.compaction) this.compaction.afterAgentSettled(ctx);
     else this.syncSessionIndex(ctx);
+    this.refreshProjectMemorySources();
     this.refreshProjectIndex();
   }
 
@@ -1350,6 +1540,107 @@ export class Ds4ContextRuntime {
     return this.memoryManager?.listMemories(activeOnly) ?? [];
   }
 
+  projectMemorySources(): ProjectMemorySource[] {
+    return this.lastCrossSessionMemory.sources.map((source) => ({ ...source }));
+  }
+
+  setProjectMemorySourceExcluded(
+    sessionId: string,
+    excluded: boolean,
+    reason?: string,
+  ): ProjectMemorySource[] {
+    if (!this.projectMemorySynchronizer || !this.memoryManager) {
+      throw new Error("Cross-session project memory is unavailable for this session");
+    }
+    this.lastCrossSessionMemory = this.projectMemorySynchronizer.setExcluded(
+      sessionId,
+      excluded,
+      reason,
+    );
+    this.memoryManager.setCrossSessionDiagnostics(this.lastCrossSessionMemory);
+    this.lastMemory = this.memoryManager.diagnostics();
+    return this.projectMemorySources();
+  }
+
+  recordRankingFeedback(
+    candidateId: string,
+    label: RankingFeedbackLabel,
+    classification: PrivacyClassification | undefined,
+    ctx: ExtensionContext,
+    appendEntry: CustomEntryAppender,
+  ): RankingFeedbackEntry {
+    const normalizedId = candidateId.trim();
+    let features = this.rankingCandidates.get(normalizedId);
+    let resolvedId = normalizedId;
+    if (!features) {
+      const matches = [...this.rankingCandidates].filter(([id]) => id.endsWith(`:${normalizedId}`));
+      if (matches.length === 1) {
+        resolvedId = matches[0]?.[0] ?? normalizedId;
+        features = matches[0]?.[1];
+      }
+    }
+    if (!features) {
+      throw new Error(`Ranking candidate ${candidateId} is not available in the latest managed context`);
+    }
+    if (!this.session) throw new Error("Canonical session identity is unavailable");
+    const feedback = createRankingFeedback({
+      feedbackId: this.idGenerator(),
+      createdAt: this.now(),
+      repositoryIdentity: this.session.projectPath,
+      candidateKey: resolvedId,
+      label,
+      classification: classification ?? this.config.privacy.defaultClassification,
+      features,
+    });
+    appendEntry(RANKING_FEEDBACK_CUSTOM_ENTRY_TYPE, feedback);
+    this.syncSessionIndex(ctx);
+    this.logger.info("ranking.feedback_recorded", {
+      feedbackId: feedback.feedbackId,
+      label,
+      classification: feedback.classification,
+    });
+    return feedback;
+  }
+
+  trainRankingModel(ctx: ExtensionContext): RankingTrainingResult {
+    if (!this.rankingModelPath) throw new Error("Ranking model path is unavailable");
+    const projection = projectRankingLabels(
+      ctx.sessionManager.getBranch(),
+      this.config.ranking.maxTrainingSamples,
+    );
+    const model = trainLearnedRankingModel(projection.samples, {
+      createdAt: this.now(),
+      minimumSamples: this.config.ranking.minimumTrainingSamples,
+    });
+    saveLearnedRankingModel(this.rankingModelPath, model);
+    this.rankingModel = model;
+    this.rankingWarnings = [...projection.warnings];
+    this.lastRanking = rankCandidates([], {
+      mode: this.config.ranking.mode,
+      model,
+      now: this.now,
+      maxLatencyMs: this.config.ranking.maxLatencyMs,
+    }).diagnostics;
+    if (this.config.ranking.mode === "off") {
+      this.lastRanking = { ...this.lastRanking, modelId: model.modelId };
+    }
+    this.logger.info("ranking.model_trained", {
+      modelId: model.modelId,
+      samples: model.training.sampleCount,
+      repositories: model.training.repositoryCount,
+      promoted: Boolean(model.promotion?.eligible),
+    });
+    return {
+      modelId: model.modelId,
+      modelPath: this.rankingModelPath,
+      sampleCount: model.training.sampleCount,
+      positiveSamples: model.training.positiveSamples,
+      negativeSamples: model.training.negativeSamples,
+      repositoryCount: model.training.repositoryCount,
+      warnings: [...projection.warnings],
+    };
+  }
+
   createPin(
     input: CreatePinInput,
     ctx: ExtensionContext,
@@ -1550,33 +1841,66 @@ export class Ds4ContextRuntime {
   }
 
   private initializeMemory(ctx: ExtensionContext): void {
+    this.projectMemorySynchronizer = undefined;
+    this.lastCrossSessionMemory = disabledCrossSessionMemoryDiagnostics();
     if (this.config.context.mode !== "managed" || !this.config.memory.enabled) {
       this.memoryManager = undefined;
       this.lastMemory = disabledMemoryDiagnostics();
       return;
     }
-    if (!this.database || !this.session?.sessionFile) {
+    if (!this.database || !this.indexer || !this.session?.sessionFile) {
       this.memoryManager = undefined;
       this.lastMemory = disabledMemoryDiagnostics("Memory and pins require a persisted Pi session");
       return;
     }
     try {
+      const projectPath = canonicalProjectPath(this.session.projectPath);
+      const projectTrusted = ctx.isProjectTrusted();
       this.memoryManager = new MemoryManager(
         this.database.memory,
         this.config.memory,
         this.config.context.maxPinnedTokens,
         this.config.context.maxMemoryTokens,
         this.session.sessionId,
-        canonicalProjectPath(this.session.projectPath),
-        ctx.isProjectTrusted(),
+        projectPath,
+        projectTrusted,
         this.now,
         this.idGenerator,
       );
       if (existsSync(this.session.sessionFile)) this.reconcileMemory(ctx);
       else this.lastMemory = this.memoryManager.diagnostics();
+
+      if (this.config.memory.crossSession && projectTrusted) {
+        this.projectMemorySynchronizer = new ProjectMemorySynchronizer(
+          this.database.memory,
+          this.indexer,
+          {
+            projectPath,
+            activeSessionId: this.session.sessionId,
+            activeSessionFile: this.session.sessionFile,
+            maxSessions: this.config.memory.maxProjectSessions,
+            logger: this.logger,
+            now: this.now,
+          },
+        );
+        this.refreshProjectMemorySources();
+      } else {
+        const reason = this.config.memory.crossSession
+          ? "Cross-session project memory requires a trusted Pi project"
+          : undefined;
+        this.lastCrossSessionMemory = disabledCrossSessionMemoryDiagnostics(reason);
+        this.memoryManager.setCrossSessionDiagnostics(this.lastCrossSessionMemory);
+        this.lastMemory = this.memoryManager.diagnostics();
+      }
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
       this.memoryManager = undefined;
+      this.projectMemorySynchronizer = undefined;
+      this.lastCrossSessionMemory = {
+        ...disabledCrossSessionMemoryDiagnostics(message),
+        enabled: this.config.memory.crossSession,
+        status: "failed",
+      };
       this.lastMemory = {
         ...disabledMemoryDiagnostics(message),
         status: "failed",
@@ -1600,7 +1924,7 @@ export class Ds4ContextRuntime {
     ];
     if (sourceEntryIds.some((entryId) => !this.database?.sessionIndex.hasEntry(this.session!.sessionId, entryId))) {
       if (!this.indexer) throw new Error("Session index is unavailable for memory mutation replay");
-      this.session = snapshotSession(ctx);
+      this.session = this.snapshotCanonicalSession(ctx);
       const rebuilt = this.indexer.sync(this.session, true);
       this.lastIndexResult = rebuilt;
       this.lastIndexError = undefined;
@@ -1622,6 +1946,24 @@ export class Ds4ContextRuntime {
     });
   }
 
+  private refreshProjectMemorySources(): void {
+    if (!this.projectMemorySynchronizer || !this.memoryManager) return;
+    try {
+      this.lastCrossSessionMemory = this.projectMemorySynchronizer.sync();
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      this.lastCrossSessionMemory = {
+        ...this.lastCrossSessionMemory,
+        enabled: true,
+        status: "failed",
+        warnings: [...new Set([...this.lastCrossSessionMemory.warnings, message])],
+      };
+      this.logger.warn("project_memory.failed", { error: message });
+    }
+    this.memoryManager.setCrossSessionDiagnostics(this.lastCrossSessionMemory);
+    this.lastMemory = this.memoryManager.diagnostics();
+  }
+
   private requireMemoryManager(): MemoryManager {
     if (!this.memoryManager) throw new Error("DS4 memory and pins are unavailable for this session");
     return this.memoryManager;
@@ -1694,6 +2036,102 @@ export class Ds4ContextRuntime {
     });
   }
 
+  private rememberPriorRankingSelection(candidateId: string, selected: number): void {
+    this.priorRankingSelection.delete(candidateId);
+    this.priorRankingSelection.set(candidateId, selected);
+    while (this.priorRankingSelection.size > MAX_PRIOR_RANKING_OUTCOMES) {
+      const oldest = this.priorRankingSelection.keys().next().value as string | undefined;
+      if (!oldest) break;
+      this.priorRankingSelection.delete(oldest);
+    }
+  }
+
+  private initializeRanking(): void {
+    this.rankingModel = undefined;
+    this.rankingWarnings = [];
+    this.rankingModelPath = resolveRankingModelPath(
+      this.config.ranking.modelPath,
+      this.dependencies.agentDir,
+      this.dependencies.homeDir,
+    );
+    this.lastRanking = disabledRankingDiagnostics(this.config.ranking.mode);
+    if (this.config.ranking.mode === "off") return;
+    const loaded = loadLearnedRankingModel(this.rankingModelPath);
+    if (loaded.model) {
+      this.rankingModel = loaded.model;
+      this.lastRanking = rankCandidates([], {
+        mode: this.config.ranking.mode,
+        model: loaded.model,
+        now: this.now,
+        maxLatencyMs: this.config.ranking.maxLatencyMs,
+      }).diagnostics;
+      if (this.config.ranking.mode === "active" && !loaded.model.promotion?.eligible) {
+        this.rankingWarnings.push("Configured active model has not passed the promotion gate; static ranking remains active");
+      }
+      return;
+    }
+    this.rankingWarnings.push(`Learned ranking unavailable: ${loaded.error ?? "model load failed"}`);
+  }
+
+  private initializeSemanticRetrieval(): void {
+    this.semanticIndex = undefined;
+    this.semanticFallbackReason = undefined;
+    if (!this.config.retrieval.semantic || !this.database) return;
+
+    const configured = this.config.retrieval.embedding;
+    const expectedDestination = configured.mode;
+    const matchesConfiguration = (port: EmbeddingPort): boolean =>
+      port.identity.provider === configured.provider
+      && port.identity.model === configured.model
+      && port.identity.dimensions === configured.dimensions
+      && port.identity.destination === expectedDestination;
+
+    let port = this.dependencies.embeddingPort;
+    if (port && !matchesConfiguration(port)) port = undefined;
+    if (!port && configured.mode === "local"
+      && configured.provider === "ds4-local"
+      && configured.model === "feature-hash-v1") {
+      port = new LocalFeatureHashEmbedding(configured.dimensions);
+    }
+    if (configured.mode === "remote") {
+      const profile = `${configured.provider}/${configured.model}`;
+      if (!configured.remoteProfiles.includes(profile)) {
+        this.semanticFallbackReason = "remote embedding profile lacks explicit consent";
+        return;
+      }
+      if (!this.config.privacy.enabled || !this.privacyEngine) {
+        this.semanticFallbackReason = "remote embedding requires privacy filtering";
+        return;
+      }
+    }
+    if (!port) {
+      this.semanticFallbackReason = `embedding model unavailable: ${configured.provider}/${configured.model}`;
+      return;
+    }
+
+    this.semanticIndex = new SemanticEmbeddingIndex(
+      this.database.embeddings,
+      port,
+      {
+        maxSources: configured.maxSources,
+        candidatePool: configured.candidatePool,
+        batchSize: configured.batchSize,
+        queryCacheSize: configured.queryCacheSize,
+        timeoutMs: configured.timeoutMs,
+      },
+      (text) => {
+        if (port!.identity.destination === "local") return text;
+        const sanitized = this.privacyEngine?.sanitizeText(text, port!.identity.provider);
+        if (!sanitized || sanitized.classification === "local-only" || sanitized.blockedBlocks > 0) {
+          return undefined;
+        }
+        return sanitized.value;
+      },
+      this.now,
+      this.now,
+    );
+  }
+
   private initializeProjectKnowledge(ctx: ExtensionContext): void {
     if (!this.config.project.enabled) {
       this.lastProject = emptyProjectDiagnostics(
@@ -1713,6 +2151,23 @@ export class Ds4ContextRuntime {
       );
       return;
     }
+    if (isBroadProjectRoot(ctx.cwd, this.dependencies.homeDir ?? homedir())) {
+      this.lastProject = {
+        ...emptyProjectDiagnostics(
+          "disabled",
+          true,
+          this.config.context.maxProjectTokens,
+          this.config.project.maxResults,
+        ),
+        projectPath: resolve(ctx.cwd),
+        fallbackReason: "Project indexing is skipped for filesystem roots and the user home directory",
+      };
+      this.logger.warn("project_index.skipped", {
+        reason: "broad-root",
+        projectPath: resolve(ctx.cwd),
+      });
+      return;
+    }
     if (!this.database) return;
 
     try {
@@ -1722,8 +2177,22 @@ export class Ds4ContextRuntime {
         this.config.project,
         this.config.context.maxProjectTokens,
         this.now,
+        {
+          enabled: this.config.retrieval.semantic,
+          ...(this.semanticIndex ? { index: this.semanticIndex } : {}),
+          ...(this.semanticFallbackReason ? { fallbackReason: this.semanticFallbackReason } : {}),
+        },
       );
-      const sync = this.projectKnowledge.sync();
+      const sync = this.syncProjectKnowledge();
+      if (!sync) {
+        this.projectRefreshPending = true;
+        this.lastProject = this.projectKnowledge.diagnostics(
+          "ready",
+          "Project indexing deferred while another Pi session indexes this project",
+        );
+        this.logger.debug("project_index.deferred", { projectPath: this.projectKnowledge.projectPath });
+        return;
+      }
       this.lastProject = this.projectKnowledge.diagnostics();
       this.logger.info("project_index.opened", {
         files: sync.discoveredFiles,
@@ -1751,15 +2220,78 @@ export class Ds4ContextRuntime {
     }
   }
 
+  private withProjectIndexLease<T>(
+    operation: (checkpoint: () => void) => T,
+  ): { acquired: true; value: T } | { acquired: false } {
+    const database = this.database;
+    const project = this.projectKnowledge;
+    if (!project || !database) return { acquired: false };
+    const durationMs = this.config.storage.projectIndexLeaseMs;
+    const lease = database.leases.acquire(
+      "project-index",
+      project.projectPath,
+      this.projectLeaseOwnerId,
+      this.now(),
+      durationMs,
+    );
+    if (!lease) return { acquired: false };
+    let activeLease = lease;
+
+    let renewAt = this.now() + Math.max(1_000, Math.floor(durationMs / 3));
+    const checkpoint = (): void => {
+      const current = this.now();
+      if (current < renewAt) {
+        if (!database.leases.isHeld(activeLease, current)) {
+          throw new Error("Project indexing lease expired or was superseded");
+        }
+        return;
+      }
+      const renewed = database.leases.renew(activeLease, current, durationMs);
+      if (!renewed) throw new Error("Project indexing lease expired or was superseded");
+      activeLease = renewed;
+      renewAt = current + Math.max(1_000, Math.floor(durationMs / 3));
+    };
+
+    try {
+      return { acquired: true, value: operation(checkpoint) };
+    } finally {
+      try {
+        database.leases.release(activeLease);
+      } catch (error) {
+        this.logger.warn("project_index.lease_release_failed", {
+          error: error instanceof Error ? error.message : String(error),
+        });
+      }
+    }
+  }
+
+  private syncProjectKnowledge(
+    force = false,
+  ): ReturnType<ProjectKnowledgeManager["sync"]> | undefined {
+    const project = this.projectKnowledge;
+    if (!project) return undefined;
+    const result = this.withProjectIndexLease((checkpoint) => project.sync(force, checkpoint));
+    return result.acquired ? result.value : undefined;
+  }
+
   private refreshProjectIndexIfPending(): void {
     if (this.projectRefreshPending) this.refreshProjectIndex();
   }
 
   private refreshProjectIndex(force = false): void {
     if (!this.projectKnowledge) return;
-    this.projectRefreshPending = false;
     try {
-      const sync = this.projectKnowledge.sync(force);
+      const sync = this.syncProjectKnowledge(force);
+      if (!sync) {
+        this.projectRefreshPending = true;
+        this.lastProject = this.projectKnowledge.diagnostics(
+          "ready",
+          "Project indexing deferred while another Pi session indexes this project",
+        );
+        this.logger.debug("project_index.deferred", { projectPath: this.projectKnowledge.projectPath });
+        return;
+      }
+      this.projectRefreshPending = false;
       this.lastProject = this.projectKnowledge.diagnostics();
       this.logger.debug("project_index.synced", {
         mode: sync.mode,
@@ -1784,7 +2316,16 @@ export class Ds4ContextRuntime {
     if (!this.projectKnowledge) return this.lastProject;
     if (!requestText) return this.projectKnowledge.diagnostics();
     try {
-      const diagnostics = this.projectKnowledge.retrieve(requestText, this.now(), maxTokens);
+      const project = this.projectKnowledge;
+      const timestamp = this.now();
+      const guarded = this.withProjectIndexLease((checkpoint) =>
+        project.retrieve(requestText, timestamp, maxTokens, checkpoint));
+      const diagnostics = guarded.acquired
+        ? guarded.value
+        : project.retrieve(requestText, timestamp, maxTokens, false);
+      if (!guarded.acquired) {
+        this.logger.debug("project_retrieval.validation_deferred", { projectPath: project.projectPath });
+      }
       this.logger.debug("project_retrieval.completed", {
         candidates: diagnostics.candidateCount,
         selected: diagnostics.selected.length,
@@ -1849,14 +2390,20 @@ export class Ds4ContextRuntime {
     }
   }
 
+  private snapshotCanonicalSession(ctx: ExtensionContext): PiSessionSnapshot {
+    const snapshot = snapshotSession(ctx);
+    return { ...snapshot, projectPath: canonicalProjectPath(snapshot.projectPath) };
+  }
+
   syncSessionIndex(ctx: ExtensionContext): SessionIndexResult | undefined {
     if (!this.indexer || this.phase === "disabled" || this.phase === "degraded" || this.phase === "closed") {
       return undefined;
     }
 
-    this.session = snapshotSession(ctx);
+    this.session = this.snapshotCanonicalSession(ctx);
     try {
       this.lastIndexResult = this.indexer.sync(this.session);
+      this.retrievalEngine?.syncSemantic(this.session.sessionId);
       this.lastIndexError = undefined;
       return this.lastIndexResult;
     } catch (error) {
@@ -1871,7 +2418,7 @@ export class Ds4ContextRuntime {
 
   rebuildIndex(ctx: ExtensionContext): SessionIndexResult {
     if (!this.indexer || !this.database) throw new Error("Context database is unavailable");
-    this.session = snapshotSession(ctx);
+    this.session = this.snapshotCanonicalSession(ctx);
     if (!this.session.sessionFile) throw new Error("Ephemeral Pi sessions do not have a JSONL index");
 
     const result = this.indexer.sync(this.session, true);
@@ -1879,12 +2426,100 @@ export class Ds4ContextRuntime {
     this.lastIndexError = undefined;
     this.refreshProjectIndex(true);
     this.reconcileMemory(ctx, true);
+    this.refreshProjectMemorySources();
     this.rebuildArtifacts(ctx);
     return result;
   }
 
+  private flushContextQuality(): void {
+    const pending = this.pendingQuality.splice(0);
+    for (const item of pending) {
+      this.recordContextQuality(item.manifest, item.contextConfig, item.recordedAt);
+    }
+  }
+
+  private recordContextQuality(
+    manifest: ContextManifest,
+    contextConfig: Ds4ContextConfig["context"],
+    recordedAt: number,
+  ): void {
+    if (!this.config.quality.enabled || !this.database) return;
+    try {
+      const sample = evaluateManifestQuality(manifest, {
+        pin: contextConfig.maxPinnedTokens,
+        memory: contextConfig.maxMemoryTokens,
+        retrieval: contextConfig.maxRetrievedHistoryTokens,
+        project: contextConfig.maxProjectTokens,
+        summary: contextConfig.maxSummaryTokens,
+        recent: manifest.planning?.recentTailTokenLimit ?? contextConfig.recentTailTokens,
+      });
+      this.database.quality.save(sample, recordedAt, this.config.quality.maxSamples);
+      this.lastQualityError = undefined;
+    } catch (error) {
+      this.lastQualityError = error instanceof Error ? error.message : String(error);
+      this.logger.warn("quality.sample_ignored", { error: this.lastQualityError });
+    }
+  }
+
+  private contextQualityDiagnostics(): ContextQualityDiagnostics {
+    const empty = disabledContextQualityDiagnostics();
+    if (!this.config.quality.enabled) return empty;
+    if (!this.database) {
+      return {
+        ...empty,
+        enabled: true,
+        ...(this.lastQualityError ? { lastError: this.lastQualityError } : {}),
+      };
+    }
+    try {
+      const stored = this.database.quality.aggregate(this.config.quality.maxSamples);
+      return {
+        enabled: true,
+        metricsVersion: empty.metricsVersion,
+        storedSamples: stored.samples.length,
+        ignoredSamples: stored.ignoredSamples,
+        aggregate: stored.aggregate,
+        timing: stored.timing,
+        ...(this.lastQualityError ? { lastError: this.lastQualityError } : {}),
+      };
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      return { ...empty, enabled: true, lastError: message };
+    }
+  }
+
+  private runtimeRankingDiagnostics(ctx: ExtensionContext): RuntimeRankingDiagnostics {
+    try {
+      const projection = projectRankingLabels(
+        ctx.sessionManager.getBranch(),
+        this.config.ranking.maxTrainingSamples,
+      );
+      return {
+        ...this.lastRanking,
+        ...(this.rankingModelPath ? { modelPath: this.rankingModelPath } : {}),
+        modelLoaded: Boolean(this.rankingModel),
+        promoted: Boolean(this.rankingModel?.promotion?.eligible),
+        trainingSamples: projection.samples.length,
+        malformedFeedback: projection.malformedEntries,
+        duplicateFeedback: projection.duplicateEntries,
+        warnings: [...new Set([...this.rankingWarnings, ...projection.warnings])],
+      };
+    } catch (error) {
+      return {
+        ...this.lastRanking,
+        ...(this.rankingModelPath ? { modelPath: this.rankingModelPath } : {}),
+        modelLoaded: Boolean(this.rankingModel),
+        promoted: Boolean(this.rankingModel?.promotion?.eligible),
+        trainingSamples: 0,
+        malformedFeedback: 0,
+        duplicateFeedback: 0,
+        warnings: [...this.rankingWarnings, error instanceof Error ? error.message : String(error)],
+      };
+    }
+  }
+
   diagnostics(ctx: ExtensionContext): RuntimeDiagnostics {
-    const currentSession = this.session ?? snapshotSession(ctx);
+    const currentSession = this.session ?? this.snapshotCanonicalSession(ctx);
     if (this.artifactManager) {
       this.lastArtifacts = this.artifactManager.diagnostics(
         new Set(ctx.sessionManager.getBranch().map((entry) => entry.id)),
@@ -1906,6 +2541,7 @@ export class Ds4ContextRuntime {
       phase: this.phase,
       enabled: this.config.enabled,
       contextMode: this.config.context.mode,
+      adapter: negotiatePiRuntimeCapabilities(),
       session: currentSession,
       ...(ctx.model ? { model: { provider: ctx.model.provider, id: ctx.model.id } } : {}),
       ...(this.databasePath ? { databasePath: this.databasePath } : {}),
@@ -1919,6 +2555,8 @@ export class Ds4ContextRuntime {
       privacy: this.lastPrivacy,
       ...(this.lastModelAwareness ? { modelAwareness: this.lastModelAwareness } : {}),
       nativeContinuation: this.nativeContinuation.diagnostics(),
+      quality: this.contextQualityDiagnostics(),
+      ranking: this.runtimeRankingDiagnostics(ctx),
       artifacts: this.lastArtifacts,
       compaction: this.getCompactionDiagnostics(ctx),
       ...(this.lastIndexResult ? { lastIndexResult: this.lastIndexResult } : {}),
@@ -1944,6 +2582,7 @@ export class Ds4ContextRuntime {
 
   shutdown(ctx?: ExtensionContext): void {
     if (ctx) this.syncSessionIndex(ctx);
+    this.flushContextQuality();
     this.nativeContinuation.invalidate("session-shutdown");
     this.closeDatabase();
     if (ctx) this.setStatus(ctx, undefined);
@@ -1976,6 +2615,8 @@ export class Ds4ContextRuntime {
       this.projectKnowledge = undefined;
       this.projectRefreshPending = false;
       this.memoryManager = undefined;
+      this.projectMemorySynchronizer = undefined;
+      this.lastCrossSessionMemory = disabledCrossSessionMemoryDiagnostics();
       this.lastMemoryMutationSignature = undefined;
       this.artifactManager = undefined;
       this.indexer = undefined;