@duckmind/dm-darwin-x64 0.66.4 → 0.66.6

package/extensions/dm-lens/clients/project-snapshot.js

@@ -7,6 +7,8 @@ import { writeFileAtomic } from "./atomic-write.js";
 import { BoundedFifoMap } from "./bounded-cache.js";
 import { getProjectDataDir } from "./file-utils.js";
 import { incrementDegradationCount } from "./degradation-ledger.js";
+import { withGenerationLockSync } from "./generation-lock.js";
+import { isStaleStageFile } from "./instance-reaper.js";
 import { readJsonCache } from "./json-cache-read.js";
 import { logLatency } from "./latency-logger.js";
 import { normalizeMapKey } from "./path-utils.js";
@@ -14,8 +16,12 @@ import { fingerprintProjectSnapshotJson } from "./project-snapshot-fingerprint.j
 import {
   detectProjectConventions
 } from "./project-conventions.js";
+import {
+  readWorkerHeapStatistics
+} from "./persist-worker-stats.js";
 import {
   deserializeWordIndex,
+  recordPersistedWordIndexWireBytes,
   serializeWordIndex
 } from "./word-index.js";
 export const PROJECT_SNAPSHOT_VERSION = 2;
@@ -39,8 +45,25 @@ export function getProjectSnapshotLegacyPath(cwd) {
 export function getProjectSnapshotMetaPath(cwd) {
   return path.join(getProjectDataDir(cwd), "cache", "project-snapshot.meta.json");
 }
-export function isProjectSnapshotFresh(snapshot, currentProjectSeq) {
-  return !!snapshot && snapshot.version === PROJECT_SNAPSHOT_VERSION && snapshot.seq === currentProjectSeq;
+const INCOMPLETE_STORED_SEQ = -2;
+function storedSeq(snapshot) {
+  return snapshot.incomplete ? { seq: INCOMPLETE_STORED_SEQ, incompleteSeq: snapshot.seq } : { seq: snapshot.seq };
+}
+function storedSnapshot(snapshot) {
+  if (!snapshot.incomplete)
+    return snapshot;
+  const stored = { ...snapshot };
+  delete stored.incomplete;
+  return { ...stored, ...storedSeq(snapshot) };
+}
+function readStoredSeq(stored) {
+  if (typeof stored.incompleteSeq === "number") {
+    return { seq: stored.incompleteSeq, incomplete: true };
+  }
+  return typeof stored.seq === "number" ? { seq: stored.seq } : null;
+}
+export function isProjectSnapshotFresh(snapshot, currentProjectSeq, unlockedThrough = 0) {
+  return !!snapshot && snapshot.version === PROJECT_SNAPSHOT_VERSION && !snapshot.incomplete && (snapshot.logEntries ?? 0) >= unlockedThrough && snapshot.seq === currentProjectSeq;
 }
 function parseSnapshot(value) {
   if (!value || typeof value !== "object")
@@ -52,7 +75,8 @@ function parseSnapshot(value) {
     return null;
   if (typeof snapshot.generatedAt !== "string")
     return null;
-  if (typeof snapshot.seq !== "number")
+  const stored = readStoredSeq(value);
+  if (!stored)
     return null;
   if (!Array.isArray(snapshot.cachedExports))
     return null;
@@ -60,11 +84,13 @@ function parseSnapshot(value) {
     version: PROJECT_SNAPSHOT_VERSION,
     projectRoot: snapshot.projectRoot,
     generatedAt: snapshot.generatedAt,
-    seq: snapshot.seq,
+    seq: stored.seq,
     files: snapshot.files ?? {},
     symbols: snapshot.symbols ?? {},
     reverseDeps: snapshot.reverseDeps ?? {},
     cachedExports: snapshot.cachedExports.filter((entry) => Array.isArray(entry) && typeof entry[0] === "string" && typeof entry[1] === "string"),
+    ...stored.incomplete ? { incomplete: true } : {},
+    logEntries: typeof snapshot.logEntries === "number" ? snapshot.logEntries : undefined,
     sequenceIndex: parseSequenceIndex(snapshot.sequenceIndex),
     wordIndex: snapshot.wordIndex,
     projectRulesScan: snapshot.projectRulesScan,
@@ -79,14 +105,17 @@ function parseSnapshotMeta(value) {
   const meta = value;
   if (typeof meta.version !== "number")
     return null;
-  if (typeof meta.seq !== "number")
+  const stored = readStoredSeq(value);
+  if (!stored)
     return null;
   return {
     timestamp: typeof meta.timestamp === "string" ? meta.timestamp : "",
     version: meta.version,
-    seq: meta.seq,
+    seq: stored.seq,
     fingerprint: typeof meta.fingerprint === "string" && /^[a-f0-9]{64}$/.test(meta.fingerprint) ? meta.fingerprint : undefined,
     gzBytes: typeof meta.gzBytes === "number" && Number.isInteger(meta.gzBytes) && meta.gzBytes > 0 ? meta.gzBytes : undefined,
+    ...stored.incomplete ? { incomplete: true } : {},
+    logEntries: typeof meta.logEntries === "number" ? meta.logEntries : undefined,
     sequenceIndex: parseSequenceIndex(meta.sequenceIndex)
   };
 }
@@ -94,8 +123,8 @@ export function readProjectSnapshotMeta(cwd) {
   const meta = readJsonCache(getProjectSnapshotMetaPath(cwd), (parsed) => parseSnapshotMeta(parsed) ?? undefined);
   return meta ?? null;
 }
-export function isProjectSnapshotMetaStale(meta, currentProjectSeq) {
-  return meta.version !== PROJECT_SNAPSHOT_VERSION || meta.seq !== currentProjectSeq;
+export function isProjectSnapshotMetaStale(meta, currentProjectSeq, unlockedThrough = 0) {
+  return meta.version !== PROJECT_SNAPSHOT_VERSION || meta.incomplete === true || (meta.logEntries ?? 0) < unlockedThrough || meta.seq !== currentProjectSeq;
 }
 export const SNAPSHOT_PARSE_CACHE_MAX = 4;
 const SNAPSHOT_PARSE_CACHE_MAX_BYTES = 24 * 1024 * 1024;
@@ -341,13 +370,16 @@ function parseExportsAndRulesOnly(json) {
   const parsed = JSON.parse(narrowed);
   if (parsed.version !== PROJECT_SNAPSHOT_VERSION)
     return null;
-  if (typeof parsed.seq !== "number")
+  const stored = readStoredSeq(parsed);
+  if (!stored)
     return null;
   if (!Array.isArray(parsed.cachedExports))
     return null;
   return {
     version: PROJECT_SNAPSHOT_VERSION,
-    seq: parsed.seq,
+    seq: stored.seq,
+    ...stored.incomplete ? { incomplete: true } : {},
+    logEntries: typeof parsed.logEntries === "number" ? parsed.logEntries : undefined,
     cachedExports: parsed.cachedExports.filter((entry) => Array.isArray(entry) && typeof entry[0] === "string" && typeof entry[1] === "string"),
     projectRulesScan: parsed.projectRulesScan
   };
@@ -377,8 +409,22 @@ export function loadProjectSnapshotExportsAndRules(cwd) {
   if (authoritative) {
     const notSuperseded = body === null || body.mtimeMs <= authoritative.knownMtime && body.size === authoritative.knownSize;
     if (notSuperseded) {
-      const { version, seq, cachedExports, projectRulesScan } = authoritative.snapshot;
-      return { version, seq, cachedExports, projectRulesScan };
+      const {
+        version,
+        seq,
+        incomplete,
+        logEntries,
+        cachedExports,
+        projectRulesScan
+      } = authoritative.snapshot;
+      return {
+        version,
+        seq,
+        ...incomplete ? { incomplete } : {},
+        logEntries,
+        cachedExports,
+        projectRulesScan
+      };
     }
   }
   if (!body)
@@ -441,6 +487,20 @@ let _snapshotPromotionSeamForTests;
 let _lastSnapshotPersistErrorForTests;
 let _snapshotExiting = false;
 let _snapshotWorkerBodyWritesForTests = 0;
+let _snapshotPersistWorkerHeapStatistics = null;
+export function refreshProjectSnapshotPersistWorkerHeapStatistics() {
+  const worker = _snapshotPersistWorker;
+  if (!worker || typeof worker.getHeapStatistics !== "function") {
+    return Promise.resolve();
+  }
+  return readWorkerHeapStatistics(worker).then((stats) => {
+    if (_snapshotPersistWorker === worker)
+      _snapshotPersistWorkerHeapStatistics = stats;
+  }).catch(() => {});
+}
+export function getProjectSnapshotPersistWorkerHeapStatistics() {
+  return _snapshotPersistWorkerHeapStatistics;
+}
 function snapshotWorkerEnabled() {
   const raw = process.env.DM_LENS_SNAPSHOT_PERSIST_SYNC;
   return !(raw === "1" || raw === "true");
@@ -544,11 +604,12 @@ function writeProjectSnapshotMeta(metaPath, snapshot, bodyRecord) {
   writeFileAtomic(metaPath, JSON.stringify({
     timestamp: bodyRecord?.generatedAt ?? snapshot.generatedAt,
     version: snapshot.version,
-    seq: snapshot.seq,
+    ...storedSeq(snapshot),
     ...bodyRecord ? {
       fingerprint: bodyRecord.fingerprint,
       ...bodyRecord.gzBytes === undefined ? {} : { gzBytes: bodyRecord.gzBytes }
     } : {},
+    logEntries: snapshot.logEntries,
     ...snapshot.sequenceIndex ? { sequenceIndex: snapshot.sequenceIndex } : {}
   }), { bestEffort: false });
 }
@@ -601,7 +662,8 @@ function logSnapshotPersistDecision(args) {
       decision: args.decision,
       ...args.fingerprint ? { fingerprint: args.fingerprint.slice(0, 12) } : {},
       ...args.avoidedRawBytes === undefined ? {} : { avoidedRawBytes: args.avoidedRawBytes },
-      ...args.avoidedGzipBytes === undefined ? {} : { avoidedGzipBytes: args.avoidedGzipBytes }
+      ...args.avoidedGzipBytes === undefined ? {} : { avoidedGzipBytes: args.avoidedGzipBytes },
+      ...args.diskSeq === undefined ? {} : { diskSeq: args.diskSeq }
     }
   });
 }
@@ -666,6 +728,85 @@ function completeSnapshotPersist(pending) {
   _queuedSnapshotPersists.delete(pending.key);
   dispatchSnapshotPersist(queued);
 }
+const SNAPSHOT_CACHE_LOCK = { staleMs: 5000, waitMs: 500 };
+function withSnapshotCacheLock(gzPath, purpose, op) {
+  const result = withGenerationLockSync(`${gzPath}.locks`, SNAPSHOT_CACHE_LOCK, op);
+  if (!result.held) {
+    const code = result.cause?.code;
+    incrementDegradationCount({
+      kind: "project-snapshot-lock-unavailable",
+      subject: gzPath,
+      reason: `${purpose} skipped: ${code ? `lock failed (${code})` : "lock wait ran out"}`
+    });
+  }
+  return result;
+}
+function promoteSnapshotBody(pending, body, record) {
+  const dropStage = () => {
+    if ("stagePath" in body)
+      fs.rmSync(body.stagePath, { force: true });
+  };
+  const locked = withSnapshotCacheLock(pending.gzPath, "body promotion", () => {
+    const onDisk = readProjectSnapshotMeta(pending.cwd);
+    if (onDisk && onDisk.seq > pending.snapshot.seq) {
+      dropStage();
+      return onDisk.seq;
+    }
+    if ("stagePath" in body)
+      fs.renameSync(body.stagePath, pending.gzPath);
+    else
+      writeFileAtomic(pending.gzPath, body.gzip, { bestEffort: false });
+    fs.rmSync(pending.legacyPath, { force: true });
+    return finalizeProjectSnapshotMeta(pending, record);
+  });
+  if (!locked.held) {
+    dropStage();
+    recordSnapshotPersistFailure(pending, "project snapshot cache lock unavailable");
+    return false;
+  }
+  if (typeof locked.value === "boolean")
+    return locked.value;
+  deleteAuthoritativeSnapshot(pending.key);
+  logSnapshotPersistDecision({
+    cwd: pending.cwd,
+    seq: pending.snapshot.seq,
+    decision: "superseded_on_disk",
+    diskSeq: locked.value
+  });
+  return false;
+}
+function serializeSnapshotBody(snapshot) {
+  const started = performance.now();
+  const stored = storedSnapshot(snapshot);
+  const spliced = stored.wordIndex === undefined ? undefined : spliceWordIndexBody(stored);
+  const bytes = spliced?.bytes ?? new TextEncoder().encode(JSON.stringify(stored));
+  return {
+    bytes,
+    serializeMs: performance.now() - started,
+    wordIndexWireBytes: spliced?.wordIndexWireBytes
+  };
+}
+const WORD_INDEX_PLACEHOLDER = `\x00dm-lens-word-index-${process.pid}-${Date.now()}\x00`;
+const WORD_INDEX_KEY_JSON = '"wordIndex":';
+const WORD_INDEX_PLACEHOLDER_JSON = `${WORD_INDEX_KEY_JSON}${JSON.stringify(WORD_INDEX_PLACEHOLDER)}`;
+function spliceWordIndexBody(stored) {
+  const marker = WORD_INDEX_PLACEHOLDER_JSON;
+  const head = JSON.stringify({ ...stored, wordIndex: WORD_INDEX_PLACEHOLDER });
+  const at = head.indexOf(marker);
+  if (at < 0)
+    return;
+  const prefix = head.slice(0, at + WORD_INDEX_KEY_JSON.length);
+  const suffix = head.slice(at + marker.length);
+  const wordIndex = JSON.stringify(stored.wordIndex);
+  const prefixBytes = Buffer.byteLength(prefix);
+  const wordIndexWireBytes = Buffer.byteLength(wordIndex);
+  const bytes = new Uint8Array(prefixBytes + wordIndexWireBytes + Buffer.byteLength(suffix));
+  const encoder = new TextEncoder;
+  encoder.encodeInto(prefix, bytes.subarray(0, prefixBytes));
+  encoder.encodeInto(wordIndex, bytes.subarray(prefixBytes, prefixBytes + wordIndexWireBytes));
+  encoder.encodeInto(suffix, bytes.subarray(prefixBytes + wordIndexWireBytes));
+  return { bytes, wordIndexWireBytes };
+}
 function writeSnapshotBodyOnMainThread(pending, reason) {
   if (!pendingSnapshotIsCurrent(pending)) {
     completeSnapshotPersist(pending);
@@ -685,11 +826,10 @@ function writeSnapshotBodyOnMainThread(pending, reason) {
     });
   }
   try {
-    const serializeStarted = performance.now();
-    const json = JSON.stringify(pending.snapshot);
-    const serializeMs = performance.now() - serializeStarted;
-    const rawBytes = Buffer.byteLength(json);
-    const fingerprint = fingerprintProjectSnapshotJson(json, pending.snapshot.generatedAt);
+    const { bytes, serializeMs, wordIndexWireBytes } = serializeSnapshotBody(pending.snapshot);
+    pending.wordIndexWireBytes = wordIndexWireBytes;
+    const rawBytes = bytes.byteLength;
+    const fingerprint = fingerprintProjectSnapshotJson(bytes, pending.snapshot.generatedAt);
     if (pending.dedupeFingerprints.includes(fingerprint)) {
       const current = snapshotPersistBaselinesFor(pending.cwd, pending.key);
       if (current.integrity === "ok") {
@@ -716,17 +856,16 @@ function writeSnapshotBodyOnMainThread(pending, reason) {
       }
     }
     const writeStarted = performance.now();
-    const gzip = gzipSync(json);
+    const gzip = gzipSync(bytes);
     fs.mkdirSync(path.dirname(pending.gzPath), { recursive: true });
-    writeFileAtomic(pending.gzPath, gzip, { bestEffort: false });
-    fs.rmSync(pending.legacyPath, { force: true });
-    const metadataFinalized = finalizeProjectSnapshotMeta(pending, {
+    const promoted = promoteSnapshotBody(pending, { gzip }, {
       fingerprint,
       generatedAt: pending.snapshot.generatedAt,
       gzBytes: gzip.byteLength
     });
-    if (!metadataFinalized)
+    if (!promoted)
       return;
+    recordPersistedWordIndexWireBytes(pending.snapshot.wordIndex, pending.wordIndexWireBytes);
     reconcileAuthoritativeAfterWrite(pending, rawBytes);
     logSnapshotPersistSuccess(pending, fingerprint, {
       rawBytes,
@@ -798,22 +937,21 @@ function handleSnapshotWorkerResult(result) {
     return;
   }
   try {
-    fs.renameSync(result.stagePath, pending.gzPath);
-    fs.rmSync(pending.legacyPath, { force: true });
-    const metadataFinalized = finalizeProjectSnapshotMeta(pending, {
+    const promoted = promoteSnapshotBody(pending, { stagePath: result.stagePath }, {
       fingerprint: result.semanticFingerprint,
       generatedAt: pending.snapshot.generatedAt,
       gzBytes: result.gzBytes
     });
-    if (!metadataFinalized) {
+    if (!promoted) {
       completeSnapshotPersist(pending);
       return;
     }
+    recordPersistedWordIndexWireBytes(pending.snapshot.wordIndex, pending.wordIndexWireBytes);
     reconcileAuthoritativeAfterWrite(pending, result.rawBytes);
     logSnapshotPersistSuccess(pending, result.semanticFingerprint, {
       rawBytes: result.rawBytes,
       gzBytes: result.gzBytes,
-      serializeMs: result.serializeMs,
+      serializeMs: pending.serializeMs ?? result.serializeMs,
       writeMs: result.writeMs,
       offloaded: true
     });
@@ -838,18 +976,35 @@ function dispatchSnapshotPersist(pending) {
     dispatchMainThreadWriteThroughSeam(pending, "persist worker unavailable");
     return;
   }
+  let serialized;
+  try {
+    serialized = serializeSnapshotBody(pending.snapshot);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    incrementDegradationCount({
+      kind: "project-snapshot-serialize-failed",
+      subject: pending.gzPath,
+      reason: `snapshot body serialization failed: ${message}`
+    });
+    recordSnapshotPersistFailure(pending, message);
+    completeSnapshotPersist(pending);
+    return;
+  }
+  pending.serializeMs = serialized.serializeMs;
+  pending.wordIndexWireBytes = serialized.wordIndexWireBytes;
   const id = ++_snapshotWorkerRequestId;
   _snapshotWorkerRequests.set(id, pending);
   const request = {
     id,
     generation: pending.generation,
     stagePath: pending.stagePath,
-    data: pending.snapshot,
+    data: serialized.bytes,
+    generatedAt: pending.snapshot.generatedAt,
     priorFingerprints: pending.dedupeFingerprints,
     testDelayMs: undefined
   };
   try {
-    worker.postMessage(request);
+    worker.postMessage(request, [serialized.bytes.buffer]);
   } catch (err) {
     _snapshotWorkerRequests.delete(id);
     dispatchMainThreadWriteThroughSeam(pending, err instanceof Error ? err.message : String(err));
@@ -857,6 +1012,7 @@ function dispatchSnapshotPersist(pending) {
 }
 function handleSnapshotWorkerDeath(reason) {
   _snapshotPersistWorker = undefined;
+  _snapshotPersistWorkerHeapStatistics = null;
   _snapshotWorkerDisabled = true;
   const requests = [..._snapshotWorkerRequests.values()];
   _snapshotWorkerRequests.clear();
@@ -866,7 +1022,8 @@ function handleSnapshotWorkerDeath(reason) {
 function resolveSnapshotPersistWorkerPath() {
   const candidates = [
     new URL("./project-snapshot-persist-worker.js", import.meta.url),
-    new URL("./clients/project-snapshot-persist-worker.js", import.meta.url)
+    new URL("./workers/project-snapshot-persist-worker.js", import.meta.url),
+    new URL("../workers/project-snapshot-persist-worker.js", import.meta.url)
   ];
   for (const url of candidates) {
     try {
@@ -898,8 +1055,10 @@ function getSnapshotPersistWorker() {
     });
     worker.on("error", (err) => handleSnapshotWorkerDeath(err.message));
     worker.on("exit", (code) => {
-      if (_snapshotPersistWorker === worker)
+      if (_snapshotPersistWorker === worker) {
         _snapshotPersistWorker = undefined;
+        _snapshotPersistWorkerHeapStatistics = null;
+      }
       const stranded = [..._snapshotWorkerRequests.values()];
       if (stranded.length > 0) {
         _snapshotWorkerRequests.clear();
@@ -926,43 +1085,42 @@ function sweepStaleSnapshotStageFiles(cacheDir) {
   fs.readdir(cacheDir, (err, entries) => {
     if (err)
       return;
-    const ownMarker = `.stage-${process.pid}-`;
     for (const entry of entries) {
-      if (!entry.startsWith("project-snapshot.json.gz.stage-"))
-        continue;
-      if (entry.includes(ownMarker))
+      if (!isStaleStageFile(entry, "project-snapshot.json.gz.stage-"))
         continue;
       fs.rm(path.join(cacheDir, entry), { force: true }, () => {});
     }
   });
 }
+function flushSnapshotPersistsAtExit() {
+  _snapshotExiting = true;
+  const latestByKey = new Map;
+  for (const pending of _snapshotWorkerRequests.values()) {
+    latestByKey.set(pending.key, pending);
+  }
+  for (const pending of _queuedSnapshotPersists.values()) {
+    const prior = latestByKey.get(pending.key);
+    if (!prior || prior.generation <= pending.generation) {
+      latestByKey.set(pending.key, pending);
+    }
+  }
+  _snapshotWorkerRequests.clear();
+  _queuedSnapshotPersists.clear();
+  _activeSnapshotPersists.clear();
+  for (const pending of latestByKey.values()) {
+    writeSnapshotBodyOnMainThread(pending, "exit_hook");
+  }
+  _snapshotPersistWorker?.terminate();
+}
 let _snapshotExitHookInstalled = false;
 function ensureSnapshotPersistExitHook() {
   if (_snapshotExitHookInstalled)
     return;
   _snapshotExitHookInstalled = true;
-  process.once("exit", () => {
-    _snapshotExiting = true;
-    const latestByKey = new Map;
-    for (const pending of _snapshotWorkerRequests.values()) {
-      latestByKey.set(pending.key, pending);
-    }
-    for (const pending of _queuedSnapshotPersists.values()) {
-      const prior = latestByKey.get(pending.key);
-      if (!prior || prior.generation <= pending.generation) {
-        latestByKey.set(pending.key, pending);
-      }
-    }
-    _snapshotWorkerRequests.clear();
-    _queuedSnapshotPersists.clear();
-    _activeSnapshotPersists.clear();
-    for (const pending of latestByKey.values()) {
-      if (_snapshotGenerationStates.get(pending.key)?.generation !== pending.generation)
-        continue;
-      writeSnapshotBodyOnMainThread(pending, "exit_hook");
-    }
-    _snapshotPersistWorker?.terminate();
-  });
+  process.once("exit", flushSnapshotPersistsAtExit);
+}
+export function runSnapshotPersistExitFlushForTests() {
+  flushSnapshotPersistsAtExit();
 }
 export function saveProjectSnapshot(cwd, snapshot) {
   const gzPath = getProjectSnapshotPath(cwd);
@@ -981,9 +1139,12 @@ export function saveProjectSnapshot(cwd, snapshot) {
   if (_failedSnapshotPersists.delete(key)) {
     logSnapshotPersistDecision({ cwd, seq: snapshot.seq, decision: "retry" });
   }
-  if (priorPersist?.seq !== snapshot.seq) {
-    writeProjectSnapshotMeta(metaPath, snapshot);
-  }
+  withSnapshotCacheLock(gzPath, "admission meta write", () => {
+    const onDisk = readProjectSnapshotMeta(cwd);
+    if (!onDisk || onDisk.seq < snapshot.seq) {
+      writeProjectSnapshotMeta(metaPath, snapshot);
+    }
+  });
   const priorBody = resolveSnapshotBodyPath(cwd);
   const knownMtime = priorBody ? priorBody.mtimeMs : Number.NEGATIVE_INFINITY;
   const knownSize = priorBody ? priorBody.size : Number.NEGATIVE_INFINITY;
@@ -1050,6 +1211,7 @@ export function resetProjectSnapshotPersistWorkerForTests() {
   _snapshotGenerationGateEnabledForTests = true;
   _snapshotPromotionSeamForTests = undefined;
   _snapshotPersistWorker = undefined;
+  _snapshotPersistWorkerHeapStatistics = null;
   _snapshotWorkerRequests.clear();
   _snapshotGenerationStates.clear();
   _successfulSnapshotPersists.clear();
@@ -1103,16 +1265,19 @@ export function _failedSnapshotPersistKeysForTests() {
   return [..._failedSnapshotPersists.keys()];
 }
 export function buildProjectSnapshotFromRuntime(args) {
+  const incomplete = args.runtime.viewMissesLoggedEntries;
   return {
     version: PROJECT_SNAPSHOT_VERSION,
     projectRoot: normalizeMapKey(path.resolve(args.cwd)),
     generatedAt: new Date().toISOString(),
     seq: args.runtime.projectSeq,
+    ...incomplete ? { incomplete: true } : {},
+    logEntries: args.runtime.viewLogEntries,
     files: {},
     symbols: {},
     reverseDeps: {},
     cachedExports: [...args.runtime.cachedExports.entries()].sort((a, b) => a[0].localeCompare(b[0])),
-    sequenceIndex: {
+    sequenceIndex: incomplete ? undefined : {
       projectSeq: args.runtime.projectSeq,
       fileSeqByPath: args.runtime.getFileSeqEntries()
     },
@@ -1171,7 +1336,7 @@ export function saveRuntimeProjectSnapshot(args) {
       snapshot.files = existing.files ?? {};
       snapshot.symbols = existing.symbols ?? {};
       snapshot.reverseDeps = existing.reverseDeps ?? {};
-      if (!snapshot.wordIndex && existing.wordIndex && existing.seq === snapshot.seq) {
+      if (!snapshot.wordIndex && existing.wordIndex && existing.seq === snapshot.seq && (snapshot.incomplete || !existing.incomplete)) {
         snapshot.wordIndex = existing.wordIndex;
       }
     }

package/extensions/dm-lens/clients/python-environment.js

@@ -14,6 +14,9 @@ import {
   UV_WORKSPACE_MEMBERS_DIALECT,
   walkUpDirs
 } from "./path-utils.js";
+import { recordDegradationOnce } from "./degradation-ledger.js";
+import { logExtension } from "./extension-log.js";
+import { getProjectTrustState } from "./project-trust.js";
 const UV_WORKSPACE_TABLE = "tool\\.uv\\.workspace";
 async function findUvWorkspace(startDir, homeDir) {
   const resolvedStart = path.resolve(startDir);
@@ -55,16 +58,38 @@ function isUvWorkspaceMember(workspace, projectRoot) {
   if (relative.length === 0 || relative === ".." || relative.startsWith("../") || path.isAbsolute(relative)) {
     return false;
   }
-  const matches = (dialect) => (pattern) => matchesWorkspaceMemberPattern(pattern, relative, dialect);
+  const matches = (dialect) => (pattern) => matchesWorkspaceMemberPattern(pattern, relative, dialect, () => recordDegradationOnce({
+    kind: "workspace-glob-cap",
+    subject: String(pattern.length),
+    reason: "workspace glob matcher exceeded its memo-table cell cap"
+  }));
   return !workspace.exclude.some(matches(UV_WORKSPACE_EXCLUDE_DIALECT)) && workspace.members.some(matches(UV_WORKSPACE_MEMBERS_DIALECT));
 }
-export async function detectPythonEnvironment(projectRoot, homeDir = os.homedir()) {
+const PROJECT_SOURCED_PYTHON_SOURCES = new Set([
+  "project-dot-venv",
+  "project-venv",
+  "uv-workspace"
+]);
+const AMBIENT_ENVIRONMENT_VARIABLES = {
+  "uv-project-environment": "UV_PROJECT_ENVIRONMENT",
+  "virtual-env": "VIRTUAL_ENV",
+  conda: "CONDA_PREFIX"
+};
+export function hasAmbientPythonEnvironment(env = process.env) {
+  return Object.values(AMBIENT_ENVIRONMENT_VARIABLES).some((name) => Boolean(env[name]));
+}
+export async function detectPythonEnvironment(projectRoot, homeDir = os.homedir(), options = {}) {
   const root = path.resolve(projectRoot);
   const uvWorkspace = await findUvWorkspace(root, homeDir);
   const isProjectRoot = uvWorkspace?.isStartDir === true;
   const memberWorkspaceRoot = isProjectRoot && uvWorkspace.explicit && isUvWorkspaceMember(uvWorkspace, root) ? uvWorkspace.root : undefined;
   const uvEnvironmentRoot = isProjectRoot ? memberWorkspaceRoot ?? uvWorkspace.projectRoot : undefined;
   const uvProjectEnvironment = process.env.UV_PROJECT_ENVIRONMENT;
+  const allowAmbient = options.allowAmbient ?? true;
+  const isWithinProjectRoot = (candidateRoot) => {
+    const relative = path.relative(root, path.resolve(candidateRoot));
+    return relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative);
+  };
   const candidates = [
     ...uvEnvironmentRoot !== undefined && uvProjectEnvironment ? [
       {
@@ -86,6 +111,23 @@ export async function detectPythonEnvironment(projectRoot, homeDir = os.homedir(
   for (const candidate of candidates) {
     if (!candidate.root)
       continue;
+    if (!allowAmbient && candidate.source in AMBIENT_ENVIRONMENT_VARIABLES && !isWithinProjectRoot(candidate.root))
+      continue;
+    if (PROJECT_SOURCED_PYTHON_SOURCES.has(candidate.source) && getProjectTrustState() !== "trusted") {
+      if (recordDegradationOnce({
+        kind: "trust-refusal",
+        subject: `python-environment:${candidate.root}`,
+        reason: `project Python environment refused: pi project trust is ${getProjectTrustState()}`
+      })) {
+        logExtension({
+          subsystem: "project-trust",
+          level: "warn",
+          message: "project Python environment refused: mark the project trusted in pi or upgrade pi",
+          metadata: { source: candidate.source }
+        });
+      }
+      continue;
+    }
     const binDir = path.join(candidate.root, process.platform === "win32" ? "Scripts" : "bin");
     const pythonPath = path.join(binDir, process.platform === "win32" ? "python.exe" : "python");
     try {

package/extensions/dm-lens/clients/quiet-window.js

@@ -58,7 +58,8 @@ export async function runQuietWindow(deps) {
           runtime: deps.runtime,
           cwd: deps.cwd,
           sessionId: deps.sessionId,
-          ownerId: deps.ownerId
+          ownerId: deps.ownerId,
+          sessionGeneration: deps.runtime.captureSessionGeneration()
         });
       } catch (err) {
         ok = false;
@@ -87,9 +88,11 @@ export function registerBuiltinQuietWindowTasks(getRuntime) {
   if (_builtinsRegistered)
     return;
   _builtinsRegistered = true;
-  registerQuietWindowTask("cascade_carry_over_settle", async () => {
+  registerQuietWindowTask("cascade_carry_over_settle", async (context) => {
     const runtime = getRuntime();
-    await runtime.settleCascadeRuns(quietWindowWaitMs());
+    await runtime.settleCascadeRuns(quietWindowWaitMs(), {
+      generation: context?.sessionGeneration
+    });
   });
   registerQuietWindowTask("instance_registry_heartbeat", async () => {
     await updateHeartbeat(await buildHeartbeatResourcePatchBounded());