ds4-context-engine 0.3.0-alpha.5 → 0.3.0-beta.2

package/src/extension/runtime.ts

@@ -66,6 +66,11 @@ import type {
   PrivacyManifest,
   ProviderUsageManifest,
 } from "ds4-context-core/manifest/context-manifest";
+import { HARD_PERSISTED_MANIFEST_BYTES } from "ds4-context-core/manifest/context-manifest-storage";
+import {
+  unavailableStorageDiagnostics,
+  type StorageDiagnostics,
+} from "ds4-context-core/persistence/storage-diagnostics";
 import type { ExcludedContextSource, ObservedTool } from "ds4-context-core/manifest/observer";
 import {
   disabledCrossSessionMemoryDiagnostics,
@@ -133,6 +138,7 @@ import {
 } from "ds4-context-core/retrieval/retrieval-engine";
 import { currentRequestText } from "ds4-context-core/retrieval/task-descriptor";
 import { LocalFeatureHashEmbedding } from "../pi-adapter/local-embedding.ts";
+import { DatabaseProtocolError } from "ds4-context-core/persistence/database-client-lease";
 import { ContextDatabase, type DatabaseHealth, type SessionIndexStats } from "ds4-context-core/persistence/sqlite";
 import {
   activeTools,
@@ -371,6 +377,7 @@ export class Ds4ContextRuntime {
   private observation?: ContextObservation;
   private lastManifest?: ContextManifest;
   private pendingManifestId?: string;
+  private pendingManifestPersisted = false;
   private retrievalEngine?: HistoricalRetrievalEngine;
   private lastRetrieval: RetrievalDiagnostics = emptyRetrievalDiagnostics();
   private semanticIndex?: SemanticEmbeddingIndex;
@@ -440,6 +447,7 @@ export class Ds4ContextRuntime {
     this.lastIndexResult = undefined;
     this.lastManifest = undefined;
     this.pendingManifestId = undefined;
+    this.pendingManifestPersisted = false;
     this.retrievalEngine = undefined;
     this.semanticIndex = undefined;
     this.semanticFallbackReason = undefined;
@@ -1228,9 +1236,7 @@ export class Ds4ContextRuntime {
       }
       this.lastManifest = manifest;
       this.pendingManifestId = manifest.id;
-      if (this.config.diagnostics.storeContextManifest && this.session?.sessionFile) {
-        this.database?.manifests.save(manifest);
-      }
+      this.pendingManifestPersisted = this.persistManifest(manifest);
 
       this.logger.trace("context.planned", {
         sessionId: this.session?.sessionId,
@@ -1311,6 +1317,47 @@ export class Ds4ContextRuntime {
     }
   }
 
+  private persistManifest(manifest: ContextManifest): boolean {
+    if (!this.config.diagnostics.storeContextManifest || !this.session?.sessionFile || !this.database) {
+      return false;
+    }
+    try {
+      const result = this.database.manifests.save(manifest);
+      if (result.status === "skipped-oversize") {
+        this.logger.warn("context.manifest_persistence_skipped", {
+          category: "oversize",
+          sourceBytes: result.sourceBytes,
+          projectedBytes: result.projectedBytes,
+          includedCount: manifest.included.length,
+          excludedCount: manifest.excluded.length,
+          hardLimitBytes: HARD_PERSISTED_MANIFEST_BYTES,
+        });
+        return false;
+      }
+      this.logger.debug("context.manifest_persisted", {
+        manifestId: manifest.id,
+        completeness: result.completeness,
+        sourceBytes: result.sourceBytes,
+        storedBytes: result.storedBytes,
+      });
+      if (result.prunedManifests > 0) {
+        this.logger.debug("database.retention_progress", {
+          category: "context-manifest",
+          prunedRows: result.prunedManifests,
+          prunedBytes: result.prunedBytes,
+        });
+      }
+      return true;
+    } catch (error) {
+      this.logger.warn("context.manifest_persistence_skipped", {
+        category: "storage-write",
+        manifestId: manifest.id,
+        errorType: error instanceof Error ? error.name : "UnknownError",
+      });
+      return false;
+    }
+  }
+
   private updateManifestPrivacy(): void {
     if (!this.lastManifest?.privacy) return;
     const updatedPrivacy = this.privacyManifest(
@@ -1322,8 +1369,9 @@ export class Ds4ContextRuntime {
     );
     if (!updatedPrivacy) return;
     this.lastManifest = { ...this.lastManifest, privacy: updatedPrivacy };
-    if (this.config.diagnostics.storeContextManifest && this.session?.sessionFile) {
-      this.database?.manifests.save(this.lastManifest);
+    const persisted = this.persistManifest(this.lastManifest);
+    if (this.pendingManifestId === this.lastManifest.id) {
+      this.pendingManifestPersisted = this.pendingManifestPersisted || persisted;
     }
   }
 
@@ -1453,14 +1501,9 @@ export class Ds4ContextRuntime {
       ...this.lastManifest,
       nativeContinuation: { ...continuation },
     };
-    if (!this.config.diagnostics.storeContextManifest || !this.session?.sessionFile) return;
-    try {
-      this.database?.manifests.save(this.lastManifest);
-    } catch (error) {
-      this.logger.warn("native_continuation.manifest_persistence_failed", {
-        manifestId,
-        errorType: error instanceof Error ? error.name : "UnknownError",
-      });
+    const persisted = this.persistManifest(this.lastManifest);
+    if (this.pendingManifestId === manifestId) {
+      this.pendingManifestPersisted = this.pendingManifestPersisted || persisted;
     }
   }
 
@@ -1470,7 +1513,9 @@ export class Ds4ContextRuntime {
     if (record.role !== "assistant") return;
 
     const manifestId = this.pendingManifestId;
+    const manifestPersisted = this.pendingManifestPersisted;
     this.pendingManifestId = undefined;
+    this.pendingManifestPersisted = false;
     if (record.stopReason === "error" || record.stopReason === "aborted") return;
     const usage = record.usage;
     if (!usage || typeof usage !== "object") return;
@@ -1491,19 +1536,17 @@ export class Ds4ContextRuntime {
       };
     }
     const createdAt = this.now();
-    const persisted = Boolean(
-      this.config.diagnostics.storeContextManifest
-      && this.session?.sessionFile
-      && this.database,
-    );
     try {
-      if (persisted) {
-        const updated = this.database?.manifests.recordProviderUsage(
+      if (manifestPersisted && this.database) {
+        const updated = this.database.manifests.recordProviderUsage(
           manifestId,
           providerUsage,
           createdAt,
+          manifest?.modelAwareness?.calibration.estimator ?? "chars-v1",
         );
-        if (updated && this.lastManifest?.id === manifestId) this.lastManifest = updated;
+        if (!updated && manifest?.estimatedInputTokens) {
+          this.rememberVolatileCalibration(manifest, providerUsage, createdAt);
+        }
       } else if (manifest?.estimatedInputTokens) {
         this.rememberVolatileCalibration(manifest, providerUsage, createdAt);
       }
@@ -1513,7 +1556,8 @@ export class Ds4ContextRuntime {
       }
       this.logger.warn("context.usage_persistence_failed", {
         manifestId,
-        error: error instanceof Error ? error.message : String(error),
+        category: "storage-write",
+        errorType: error instanceof Error ? error.name : "UnknownError",
       });
     }
     this.logger.debug("context.actual_usage_recorded", {
@@ -2913,6 +2957,11 @@ export class Ds4ContextRuntime {
     return this.database.health();
   }
 
+  storageDiagnostics(): StorageDiagnostics {
+    return this.database?.storageDiagnostics(this.session?.projectPath)
+      ?? unavailableStorageDiagnostics();
+  }
+
   shutdown(ctx?: ExtensionContext): void {
     if (ctx) this.syncSessionIndex(ctx);
     this.flushContextQuality();
@@ -2931,8 +2980,16 @@ export class Ds4ContextRuntime {
     this.nativeContinuation.invalidate("runtime-degraded");
     this.closeDatabase();
     this.phase = "degraded";
-    this.lastError = error instanceof Error ? error.message : String(error);
-    this.logger.error("runtime.degraded", { stage, error: this.lastError });
+    const maintenanceActive = error instanceof DatabaseProtocolError && error.category === "maintenance-active";
+    const errorType = error instanceof Error ? error.name : "UnknownError";
+    this.lastError = maintenanceActive
+      ? "Database maintenance is active (category=maintenance-active)"
+      : `Database startup unavailable (category=storage-unavailable; errorType=${errorType})`;
+    if (maintenanceActive) {
+      this.logger.warn("database.maintenance_active", { category: "maintenance-active" });
+    } else {
+      this.logger.error("runtime.degraded", { stage, category: "storage-unavailable", errorType });
+    }
     this.setStatus(ctx, "DS4 ctx: Pi fallback");
     if (ctx.hasUI) {
       ctx.ui.notify(`DS4 Context Engine unavailable; using Pi context. ${this.lastError}`, "warning");

package/src/pi-adapter/compaction-coordinator.ts

@@ -81,6 +81,7 @@ export interface CompactionDiagnostics {
   sourcePromptTokens?: number;
   segmentCount?: number;
   aggregateCalls?: number;
+  transportRetries?: number;
   contextTokens?: number;
   softLimitTokens?: number;
   proactiveThresholdTokens?: number;
@@ -224,6 +225,7 @@ export class CompactionCoordinator {
       requestedAt,
       firstKeptEntryId: event.preparation.firstKeptEntryId,
       tokensBefore: event.preparation.tokensBefore,
+      transportRetries: 0,
     };
 
     try {
@@ -357,6 +359,7 @@ export class CompactionCoordinator {
         sourcePromptTokens: wholePlan.promptTokens,
         segmentCount: segmentPlans.length,
         aggregateCalls: aggregated.aggregateCalls,
+        transportRetries: this.state.transportRetries ?? 0,
       };
       this.dependencies.logger.debug("compaction.summary_graph_prepared", {
         activeSummaryId: activeNode.id,
@@ -369,6 +372,7 @@ export class CompactionCoordinator {
         sourcePromptTokens: wholePlan.promptTokens,
         segmentCount: segmentPlans.length,
         aggregateCalls: aggregated.aggregateCalls,
+        transportRetries: this.state.transportRetries ?? 0,
         trigger,
       });
       const activeRecord = records.at(-1);
@@ -435,6 +439,7 @@ export class CompactionCoordinator {
         ...(attempt.sourcePromptTokens !== undefined ? { sourcePromptTokens: attempt.sourcePromptTokens } : {}),
         ...(attempt.segmentCount !== undefined ? { segmentCount: attempt.segmentCount } : {}),
         ...(attempt.aggregateCalls !== undefined ? { aggregateCalls: attempt.aggregateCalls } : {}),
+        ...(attempt.transportRetries !== undefined ? { transportRetries: attempt.transportRetries } : {}),
         ...(customError ? { lastError: `Custom fallback: ${customError}` } : {}),
       };
       return;
@@ -475,6 +480,9 @@ export class CompactionCoordinator {
         : {}),
       ...(this.state.segmentCount !== undefined ? { segmentCount: this.state.segmentCount } : {}),
       ...(this.state.aggregateCalls !== undefined ? { aggregateCalls: this.state.aggregateCalls } : {}),
+      ...(this.state.transportRetries !== undefined
+        ? { transportRetries: this.state.transportRetries }
+        : {}),
     };
     this.dependencies.logger.debug("compaction.summary_graph_committed", {
       activeSummaryId: metadata.summaryId,
@@ -950,6 +958,10 @@ export class CompactionCoordinator {
       validate: this.dependencies.config.compaction.validate,
       maxSummaryTokens: this.dependencies.config.context.maxSummaryTokens,
       now: this.dependencies.now,
+      onTransportRetry: (diagnostic) => {
+        this.state.transportRetries = (this.state.transportRetries ?? 0) + 1;
+        this.dependencies.logger.debug("compaction.transport_retry", { ...diagnostic });
+      },
     });
   }
 

package/src/pi-adapter/summary-generator.ts

@@ -12,6 +12,17 @@ import {
   type SummaryValidationResult,
 } from "ds4-context-core/compaction/summary-contract";
 
+export const COMPACTION_TRANSPORT_MAX_ATTEMPTS = 3;
+const COMPACTION_TRANSPORT_RETRY_DELAYS_MS = [200, 500] as const;
+
+export interface CompactionTransportRetryDiagnostic {
+  stage: "segment" | "aggregate";
+  failedAttempt: number;
+  nextAttempt: number;
+  maxAttempts: number;
+  delayMs: number;
+}
+
 export interface GenerateValidatedSummaryInput {
   stage: "segment" | "aggregate";
   prompt: string;
@@ -23,6 +34,7 @@ export interface GenerateValidatedSummaryInput {
   event: SessionBeforeCompactEvent;
   ctx: ExtensionContext;
   now: () => number;
+  onTransportRetry?: (diagnostic: CompactionTransportRetryDiagnostic) => void;
 }
 
 export interface GeneratedSummary {
@@ -55,60 +67,134 @@ function responseErrorMessage(response: unknown): string | undefined {
   return typeof response.errorMessage === "string" ? response.errorMessage : undefined;
 }
 
-function providerFailureCategory(value: unknown): string {
+type ProviderFailureCategory =
+  | "aborted"
+  | "usage-limit"
+  | "rate-limit"
+  | "input-limit"
+  | "authentication"
+  | "transport"
+  | "provider-error";
+
+function providerFailureCategory(value: unknown): ProviderFailureCategory {
   const message = value instanceof Error
     ? value.message
     : typeof value === "string"
       ? value
       : "";
-  if (/abort|cancel/iu.test(message)) return "aborted";
+  if (value instanceof Error && value.name === "AbortError") return "aborted";
   if (/usage|quota|credit|billing/iu.test(message)) return "usage-limit";
   if (/rate|too many requests|429/iu.test(message)) return "rate-limit";
   if (/(?:context|prompt|input).{0,48}(?:exceed|limit|maximum|too (?:long|large)|tokens?)|tokens?.{0,48}(?:exceed|limit|maximum|too many)|maximum (?:context|input|prompt|length)/iu.test(message)) {
     return "input-limit";
   }
   if (/auth|credential|api.?key|permission|forbidden|401|403/iu.test(message)) return "authentication";
-  if (/timeout|network|connection|socket|dns/iu.test(message)) return "transport";
+  if (/timeout|timed out|network|connection|socket|dns|fetch failed|econn(?:reset|refused|aborted)|etimedout|eai_again|enotfound|und_err/iu.test(message)) {
+    return "transport";
+  }
+  if (/abort|cancel/iu.test(message)) return "aborted";
   return "provider-error";
 }
 
+function abortedError(): Error {
+  return new Error("Compaction summary generation aborted");
+}
+
+async function waitForTransportRetry(signal: AbortSignal, milliseconds: number): Promise<void> {
+  if (signal.aborted) throw abortedError();
+  await new Promise<void>((resolve, reject) => {
+    const onAbort = (): void => {
+      clearTimeout(timer);
+      reject(abortedError());
+    };
+    const timer = setTimeout(() => {
+      signal.removeEventListener("abort", onAbort);
+      resolve();
+    }, milliseconds);
+    signal.addEventListener("abort", onAbort, { once: true });
+  });
+}
+
+function transportFailureSuffix(category: ProviderFailureCategory, attempts: number): string {
+  return category === "transport"
+    ? `category=${category}; attempts=${attempts}`
+    : `category=${category}`;
+}
+
 export async function generateValidatedSummary(
   input: GenerateValidatedSummaryInput,
 ): Promise<GeneratedSummary> {
   const model = input.ctx.model;
   if (!model) throw new Error("Compaction summary generation requires an active model");
   const maxTokens = Math.max(1, Math.min(input.maxSummaryTokens, model.maxTokens ?? input.maxSummaryTokens));
+  const retryUsages: Usage[] = [];
   let response: Awaited<ReturnType<typeof input.ctx.modelRegistry.complete>>;
-  try {
-    response = await input.ctx.modelRegistry.complete(
-      model,
-      {
-        messages: [{
-          role: "user",
-          content: [{ type: "text", text: input.prompt }],
-          timestamp: input.now(),
-        }],
-      },
-      {
-        maxTokens,
-        signal: input.event.signal,
-        cacheRetention: "none",
-        sessionId: randomUUID(),
-      },
-    );
-  } catch (error) {
-    throw new Error(
-      `Compaction ${input.stage} request failed (category=${providerFailureCategory(error)})`,
-    );
-  }
-  if (input.event.signal.aborted) throw new Error("Compaction summary generation aborted");
-  const stopReason = responseStopReason(response);
-  if (stopReason === "length") throw new Error("Compaction summary hit the model output limit");
-  if (stopReason === "error" || stopReason === "aborted") {
-    const category = stopReason === "aborted"
-      ? "aborted"
-      : providerFailureCategory(responseErrorMessage(response));
-    throw new Error(`Compaction ${input.stage} summary stopped with ${stopReason} (category=${category})`);
+  let attempt = 0;
+  for (;;) {
+    if (input.event.signal.aborted) throw abortedError();
+    attempt++;
+    try {
+      response = await input.ctx.modelRegistry.complete(
+        model,
+        {
+          messages: [{
+            role: "user",
+            content: [{ type: "text", text: input.prompt }],
+            timestamp: input.now(),
+          }],
+        },
+        {
+          maxTokens,
+          signal: input.event.signal,
+          cacheRetention: "none",
+          sessionId: randomUUID(),
+        },
+      );
+    } catch (error) {
+      if (input.event.signal.aborted) throw abortedError();
+      const category = providerFailureCategory(error);
+      if (category !== "transport" || attempt >= COMPACTION_TRANSPORT_MAX_ATTEMPTS) {
+        throw new Error(
+          `Compaction ${input.stage} request failed (${transportFailureSuffix(category, attempt)})`,
+        );
+      }
+      const delayMs = COMPACTION_TRANSPORT_RETRY_DELAYS_MS[attempt - 1] ?? 500;
+      await waitForTransportRetry(input.event.signal, delayMs);
+      input.onTransportRetry?.({
+        stage: input.stage,
+        failedAttempt: attempt,
+        nextAttempt: attempt + 1,
+        maxAttempts: COMPACTION_TRANSPORT_MAX_ATTEMPTS,
+        delayMs,
+      });
+      continue;
+    }
+    if (input.event.signal.aborted) throw abortedError();
+    const stopReason = responseStopReason(response);
+    if (stopReason === "error") {
+      const category = providerFailureCategory(responseErrorMessage(response));
+      if (category === "transport" && attempt < COMPACTION_TRANSPORT_MAX_ATTEMPTS) {
+        retryUsages.push(response.usage);
+        const delayMs = COMPACTION_TRANSPORT_RETRY_DELAYS_MS[attempt - 1] ?? 500;
+        await waitForTransportRetry(input.event.signal, delayMs);
+        input.onTransportRetry?.({
+          stage: input.stage,
+          failedAttempt: attempt,
+          nextAttempt: attempt + 1,
+          maxAttempts: COMPACTION_TRANSPORT_MAX_ATTEMPTS,
+          delayMs,
+        });
+        continue;
+      }
+      throw new Error(
+        `Compaction ${input.stage} summary stopped with error (${transportFailureSuffix(category, attempt)})`,
+      );
+    }
+    if (stopReason === "aborted") {
+      throw new Error(`Compaction ${input.stage} summary stopped with aborted (category=aborted)`);
+    }
+    if (stopReason === "length") throw new Error("Compaction summary hit the model output limit");
+    break;
   }
   if (response.content.some((block) => block.type === "toolCall")) {
     throw new Error("Compaction summarizer attempted to call a tool");
@@ -172,7 +258,7 @@ export async function generateValidatedSummary(
       : "";
     throw new Error(`Compaction ${input.stage} summary validation failed: ${codes}${repairDiagnostics}`);
   }
-  return { content, validation, usage: response.usage };
+  return { content, validation, usage: sumUsage([...retryUsages, response.usage]) };
 }
 
 export function sumUsage(usages: readonly Usage[]): Usage {

package/src/pi-adapter/version.ts

@@ -1,4 +1,4 @@
-export const EXTENSION_VERSION = "0.3.0-alpha.5";
+export const EXTENSION_VERSION = "0.3.0-beta.2";
 export const SUPPORTED_PI_VERSION = "0.84.3";
 export const OBSERVER_PLANNER_VERSION = "observer-model-aware-v1";
 export const PLANNER_VERSION = "managed-learned-ranking-v1";