@evo-dev/core 0.0.1-alpha.2 → 0.0.1-alpha.20

package/src/evolution/evidence/session-memory/retention.ts

@@ -0,0 +1,643 @@
+import { readFile } from "node:fs/promises";
+import { createStableId, normalizeTimestamp, sha256Hex } from "../../../utils/index.ts";
+import { readEvolutionReviewSnapshot } from "../../candidates/index.ts";
+import type { EvolutionReviewSnapshot, SegmentEvolutionTriggerRecord } from "../../schema.ts";
+import { listSegmentEvolutionTriggers } from "../../triggers/index.ts";
+import { resolveSessionMemoryPaths } from "./paths.ts";
+import {
+  listSessionEvidenceSegments,
+  listSessionMemoryStates,
+  readSessionEvidenceSegment,
+  rewriteSessionRawEvents,
+  writeSessionEvidenceSegmentAtomic,
+} from "./storage.ts";
+import type { SessionEvidenceSegmentV1, SessionMemoryRawEventV1 } from "./types.ts";
+
+const DEFAULT_RETENTION_LIMIT = 20;
+const MAX_RETENTION_LIMIT = 10_000;
+const DEFAULT_LEGACY_RETENTION_DAYS = 30;
+
+export type SessionEvidenceRetentionProtectionReason =
+  | "trigger-missing"
+  | "trigger-not-terminal"
+  | "segment-review-pending"
+  | "derived-review-pending"
+  | "retention-invalid"
+  | "segment-changed";
+
+export type HistoricalSessionEvidenceRetentionProtectionReason =
+  SessionEvidenceRetentionProtectionReason;
+
+export interface PurgeExpiredSessionEvidenceResult {
+  scanned: number;
+  due: number;
+  purged: number;
+  alreadyPurged: number;
+  protected: Array<{
+    projectKey: string;
+    segmentId: string;
+    reason: SessionEvidenceRetentionProtectionReason;
+  }>;
+}
+
+export type PurgeExpiredHistoricalSessionEvidenceResult = PurgeExpiredSessionEvidenceResult;
+
+export interface SessionEvidenceRetentionInspection {
+  available: number;
+  purged: number;
+  overdue: number;
+  purgeEligible: number;
+  protected: number;
+  legacyWithoutRetention: number;
+  legacyOverdue: number;
+  legacyRawBytes: number;
+  legacyOverdueRawBytes: number;
+  expiredCapturedEventPayloads: number;
+  expiredCapturedEventRawBytes: number;
+  protectionReasons: Partial<Record<SessionEvidenceRetentionProtectionReason, number>>;
+}
+
+export type HistoricalSessionEvidenceRetentionInspection = SessionEvidenceRetentionInspection;
+
+export interface LegacySessionEvidenceRetentionMigrationResult {
+  migrated: number;
+  expiredCapturedEventPayloads: number;
+  expiredCapturedEventRawBytes: number;
+  purge: PurgeExpiredSessionEvidenceResult;
+}
+
+interface RetentionContext {
+  reviewSnapshots: Map<string, Promise<EvolutionReviewSnapshot>>;
+  triggerMaps: Map<string, Promise<Map<string, SegmentEvolutionTriggerRecord[]>>>;
+}
+
+export async function inspectSessionEvidenceRetention(input: {
+  homeDir: string;
+  projectKey?: string;
+  now?: string | Date;
+}): Promise<SessionEvidenceRetentionInspection> {
+  const result = await inspectRetention(input, () => true);
+  const expiredEvents = await expireCapturedRawEvents({ ...input, apply: false });
+  result.expiredCapturedEventPayloads = expiredEvents.payloads;
+  result.expiredCapturedEventRawBytes = expiredEvents.rawBytes;
+  return result;
+}
+
+export async function inspectHistoricalSessionEvidenceRetention(input: {
+  homeDir: string;
+  projectKey?: string;
+  now?: string | Date;
+}): Promise<HistoricalSessionEvidenceRetentionInspection> {
+  return inspectRetention(input, (segment) => segment.origin?.kind === "historical-import");
+}
+
+async function inspectRetention(
+  input: { homeDir: string; projectKey?: string; now?: string | Date },
+  include: (segment: SessionEvidenceSegmentV1) => boolean,
+): Promise<SessionEvidenceRetentionInspection> {
+  const now = normalizeTimestamp(input.now);
+  const segments = (
+    await listSessionEvidenceSegments({
+      homeDir: input.homeDir,
+      projectKey: input.projectKey,
+    })
+  ).filter(include);
+  const retentionDays = await loadLegacyRetentionDays(input.homeDir, input.projectKey);
+  const result = emptyInspection();
+  const context = createRetentionContext();
+
+  for (const segment of segments) {
+    if (segment.retention === undefined) {
+      const rawBytes = segment.rawExcerpt.stored ? segment.rawExcerpt.byteLength : 0;
+      result.legacyWithoutRetention += 1;
+      result.legacyRawBytes += rawBytes;
+      const expiresAt = legacyExpiresAt(segment, retentionDays);
+      if (Date.parse(expiresAt) <= Date.parse(now)) {
+        result.legacyOverdue += 1;
+        result.legacyOverdueRawBytes += rawBytes;
+        const protection = await findRetentionProtection({
+          homeDir: input.homeDir,
+          segment,
+          context,
+        });
+        if (protection === null) result.purgeEligible += 1;
+        else addProtection(result, protection);
+      }
+      continue;
+    }
+    if (segment.retention.rawState === "purged") {
+      result.purged += 1;
+      continue;
+    }
+    result.available += 1;
+    if (!hasValidAvailableRetention(segment)) {
+      result.overdue += 1;
+      addProtection(result, "retention-invalid");
+      continue;
+    }
+    if (Date.parse(segment.retention.expiresAt) > Date.parse(now)) continue;
+    result.overdue += 1;
+    const protection = await findRetentionProtection({
+      homeDir: input.homeDir,
+      segment,
+      context,
+    });
+    if (protection === null) result.purgeEligible += 1;
+    else addProtection(result, protection);
+  }
+  return result;
+}
+
+export async function purgeExpiredSessionEvidence(input: {
+  homeDir: string;
+  projectKey?: string;
+  now?: string | Date;
+  limit?: number;
+}): Promise<PurgeExpiredSessionEvidenceResult> {
+  return purgeRetention(input, () => true);
+}
+
+export async function purgeExpiredHistoricalSessionEvidence(input: {
+  homeDir: string;
+  projectKey?: string;
+  now?: string | Date;
+  limit?: number;
+}): Promise<PurgeExpiredHistoricalSessionEvidenceResult> {
+  return purgeRetention(input, (segment) => segment.origin?.kind === "historical-import");
+}
+
+async function purgeRetention(
+  input: { homeDir: string; projectKey?: string; now?: string | Date; limit?: number },
+  include: (segment: SessionEvidenceSegmentV1) => boolean,
+): Promise<PurgeExpiredSessionEvidenceResult> {
+  const now = normalizeTimestamp(input.now);
+  const limit = normalizeLimit(input.limit);
+  const segments = (
+    await listSessionEvidenceSegments({
+      homeDir: input.homeDir,
+      projectKey: input.projectKey,
+    })
+  )
+    .filter((segment) => include(segment) && segment.retention !== undefined)
+    .sort(
+      (left, right) =>
+        (left.retention?.expiresAt ?? "").localeCompare(right.retention?.expiresAt ?? "") ||
+        left.id.localeCompare(right.id),
+    );
+  const result: PurgeExpiredSessionEvidenceResult = {
+    scanned: 0,
+    due: 0,
+    purged: 0,
+    alreadyPurged: segments.filter((segment) => segment.retention?.rawState === "purged").length,
+    protected: [],
+  };
+  const context = createRetentionContext();
+
+  for (const segment of segments) {
+    if (result.scanned >= limit) break;
+    if (segment.retention === undefined || segment.retention.rawState === "purged") continue;
+    result.scanned += 1;
+    if (!hasValidAvailableRetention(segment)) {
+      result.due += 1;
+      result.protected.push({
+        projectKey: segment.projectKey,
+        segmentId: segment.id,
+        reason: "retention-invalid",
+      });
+      continue;
+    }
+    if (Date.parse(segment.retention.expiresAt) > Date.parse(now)) continue;
+    result.due += 1;
+
+    const protection = await findRetentionProtection({
+      homeDir: input.homeDir,
+      segment,
+      context,
+    });
+    if (protection !== null) {
+      result.protected.push({
+        projectKey: segment.projectKey,
+        segmentId: segment.id,
+        reason: protection,
+      });
+      continue;
+    }
+
+    const current = await readSessionEvidenceSegment({
+      homeDir: input.homeDir,
+      projectKey: segment.projectKey,
+      sessionKey: segment.sessionKey,
+      segmentId: segment.id,
+    });
+    if (
+      !hasValidAvailableRetention(current) ||
+      current.retention?.rawState !== "available" ||
+      current.retention.expiresAt !== segment.retention.expiresAt ||
+      !current.rawExcerpt.stored ||
+      current.rawExcerpt.sha256 !== segment.rawExcerpt.sha256 ||
+      current.lifecycle.reviewState !== segment.lifecycle.reviewState
+    ) {
+      result.protected.push({
+        projectKey: segment.projectKey,
+        segmentId: segment.id,
+        reason: "segment-changed",
+      });
+      continue;
+    }
+    const freshProtection = await findRetentionProtection({
+      homeDir: input.homeDir,
+      segment: current,
+      context: createRetentionContext(),
+    });
+    if (freshProtection !== null) {
+      result.protected.push({
+        projectKey: current.projectKey,
+        segmentId: current.id,
+        reason: freshProtection,
+      });
+      continue;
+    }
+
+    await writeSessionEvidenceSegmentAtomic({
+      homeDir: input.homeDir,
+      segment: createPurgedSegment(current, now),
+    });
+    result.purged += 1;
+  }
+
+  return result;
+}
+
+export async function migrateLegacySessionEvidenceRetention(input: {
+  homeDir: string;
+  projectKey?: string;
+  now?: string | Date;
+}): Promise<LegacySessionEvidenceRetentionMigrationResult> {
+  const now = normalizeTimestamp(input.now);
+  const retentionDays = await loadLegacyRetentionDays(input.homeDir, input.projectKey);
+  const segments = await listSessionEvidenceSegments({
+    homeDir: input.homeDir,
+    projectKey: input.projectKey,
+  });
+  let migrated = 0;
+  const context = createRetentionContext();
+  for (const segment of segments) {
+    if (segment.retention !== undefined || !segment.rawExcerpt.stored) continue;
+    const policyDays = retentionDays.get(sessionKey(segment)) ?? DEFAULT_LEGACY_RETENTION_DAYS;
+    if (Date.parse(addDays(segment.createdAt, policyDays)) > Date.parse(now)) continue;
+    const protection = await findRetentionProtection({
+      homeDir: input.homeDir,
+      segment,
+      context,
+    });
+    if (protection !== null) continue;
+    const current = await readSessionEvidenceSegment({
+      homeDir: input.homeDir,
+      projectKey: segment.projectKey,
+      sessionKey: segment.sessionKey,
+      segmentId: segment.id,
+    });
+    if (current.retention !== undefined || JSON.stringify(current) !== JSON.stringify(segment)) {
+      continue;
+    }
+    await writeSessionEvidenceSegmentAtomic({
+      homeDir: input.homeDir,
+      segment: {
+        ...current,
+        retention: {
+          policyDays,
+          expiresAt: addDays(current.createdAt, policyDays),
+          rawState: "available",
+          rawPurgedAt: null,
+          originalRawSha256: current.rawExcerpt.sha256,
+        },
+      },
+    });
+    migrated += 1;
+  }
+  const expiredEvents = await expireCapturedRawEvents({
+    homeDir: input.homeDir,
+    projectKey: input.projectKey,
+    now,
+    apply: true,
+  });
+  return {
+    migrated,
+    expiredCapturedEventPayloads: expiredEvents.payloads,
+    expiredCapturedEventRawBytes: expiredEvents.rawBytes,
+    purge: await purgeExpiredSessionEvidence({
+      homeDir: input.homeDir,
+      projectKey: input.projectKey,
+      now,
+      limit: Math.max(DEFAULT_RETENTION_LIMIT, segments.length),
+    }),
+  };
+}
+
+function emptyInspection(): SessionEvidenceRetentionInspection {
+  return {
+    available: 0,
+    purged: 0,
+    overdue: 0,
+    purgeEligible: 0,
+    protected: 0,
+    legacyWithoutRetention: 0,
+    legacyOverdue: 0,
+    legacyRawBytes: 0,
+    legacyOverdueRawBytes: 0,
+    expiredCapturedEventPayloads: 0,
+    expiredCapturedEventRawBytes: 0,
+    protectionReasons: {},
+  };
+}
+
+function addProtection(
+  result: SessionEvidenceRetentionInspection,
+  reason: SessionEvidenceRetentionProtectionReason,
+): void {
+  result.protected += 1;
+  result.protectionReasons[reason] = (result.protectionReasons[reason] ?? 0) + 1;
+}
+
+function normalizeLimit(value: number | undefined): number {
+  const limit = value ?? DEFAULT_RETENTION_LIMIT;
+  if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_RETENTION_LIMIT) {
+    throw new Error(
+      `Session Evidence retention limit must be between 1 and ${MAX_RETENTION_LIMIT}.`,
+    );
+  }
+  return limit;
+}
+
+function hasValidAvailableRetention(segment: SessionEvidenceSegmentV1): boolean {
+  const retention = segment.retention;
+  return (
+    retention !== undefined &&
+    retention.rawState === "available" &&
+    retention.rawPurgedAt === null &&
+    Number.isFinite(Date.parse(retention.expiresAt)) &&
+    /^[a-f0-9]{64}$/u.test(retention.originalRawSha256) &&
+    segment.rawExcerpt.stored &&
+    segment.rawExcerpt.byteLength === Buffer.byteLength(segment.rawExcerpt.content, "utf8") &&
+    segment.rawExcerpt.sha256 === sha256Hex(segment.rawExcerpt.content) &&
+    retention.originalRawSha256 === segment.rawExcerpt.sha256
+  );
+}
+
+async function findRetentionProtection(input: {
+  homeDir: string;
+  segment: SessionEvidenceSegmentV1;
+  context: RetentionContext;
+}): Promise<SessionEvidenceRetentionProtectionReason | null> {
+  if (
+    input.segment.lifecycle.reviewState === "unreviewed" ||
+    input.segment.lifecycle.reviewState === "deferred"
+  ) {
+    return "segment-review-pending";
+  }
+  const trigger = await findSegmentTrigger(input.homeDir, input.segment, input.context);
+  if (trigger === null) {
+    if (input.segment.reason === "session-memory-threshold") return null;
+    return "trigger-missing";
+  }
+  if (!isTerminalTrigger(trigger)) return "trigger-not-terminal";
+
+  let snapshot = input.context.reviewSnapshots.get(input.segment.projectKey);
+  if (snapshot === undefined) {
+    snapshot = readEvolutionReviewSnapshot({
+      homeDir: input.homeDir,
+      projectKey: input.segment.projectKey,
+    });
+    input.context.reviewSnapshots.set(input.segment.projectKey, snapshot);
+  }
+  return hasPendingDerivedReview(await snapshot, input.segment) ? "derived-review-pending" : null;
+}
+
+async function findSegmentTrigger(
+  homeDir: string,
+  segment: SessionEvidenceSegmentV1,
+  context: RetentionContext,
+): Promise<SegmentEvolutionTriggerRecord | null> {
+  let triggerMap = context.triggerMaps.get(segment.projectKey);
+  if (triggerMap === undefined) {
+    triggerMap = listSegmentEvolutionTriggers({ homeDir, projectKey: segment.projectKey }).then(
+      (triggers) => {
+        const map = new Map<string, SegmentEvolutionTriggerRecord[]>();
+        for (const trigger of triggers) {
+          const key = `${trigger.sessionKey}\0${trigger.segmentId}`;
+          map.set(key, [...(map.get(key) ?? []), trigger]);
+        }
+        return map;
+      },
+    );
+    context.triggerMaps.set(segment.projectKey, triggerMap);
+  }
+  const matches = (await triggerMap).get(`${segment.sessionKey}\0${segment.id}`) ?? [];
+  return matches.length === 1 ? (matches[0] ?? null) : null;
+}
+
+function isTerminalTrigger(trigger: SegmentEvolutionTriggerRecord): boolean {
+  return trigger.status === "consumed" || trigger.status === "skipped";
+}
+
+function hasPendingDerivedReview(
+  snapshot: EvolutionReviewSnapshot,
+  segment: SessionEvidenceSegmentV1,
+): boolean {
+  const runId = createStableId("segment-run", [segment.projectKey, segment.sessionKey, segment.id]);
+  if (
+    snapshot.knowledgeRecords.some(
+      (record) =>
+        record.provenance.runId === runId &&
+        !["auto-accepted", "accepted", "rejected", "deprecated", "superseded", "revoked"].includes(
+          record.reviewState,
+        ),
+    )
+  ) {
+    return true;
+  }
+  if (
+    snapshot.evosCases.some(
+      (record) =>
+        record.provenance.runId === runId &&
+        !["auto-accepted", "accepted", "rejected", "deprecated", "superseded", "revoked"].includes(
+          record.reviewState,
+        ),
+    )
+  ) {
+    return true;
+  }
+  if (
+    snapshot.repoProposals.some(
+      (proposal) =>
+        proposal.provenance.runId === runId &&
+        proposal.reviewState !== "rejected" &&
+        proposal.reviewState !== "applied" &&
+        proposal.reviewState !== "superseded",
+    )
+  ) {
+    return true;
+  }
+  return snapshot.reviewCandidates.some(
+    (candidate) =>
+      candidate.runId === runId &&
+      candidate.reviewState !== "accepted" &&
+      candidate.reviewState !== "rejected",
+  );
+}
+
+function createPurgedSegment(
+  segment: SessionEvidenceSegmentV1,
+  now: string,
+): SessionEvidenceSegmentV1 {
+  const retention = segment.retention;
+  if (retention === undefined || retention.rawState !== "available") {
+    throw new Error("Session Evidence raw body is not available for purge.");
+  }
+  return {
+    ...segment,
+    retention: {
+      ...retention,
+      rawState: "purged",
+      rawPurgedAt: now,
+    },
+    rawExcerpt: {
+      ...segment.rawExcerpt,
+      stored: false,
+      content: "",
+      byteLength: 0,
+      sha256: sha256Hex(""),
+    },
+    privacy: {
+      ...segment.privacy,
+      rawPromptStored: false,
+      rawOutputStored: false,
+      sourceContentStored: false,
+    },
+    lifecycle: {
+      ...segment.lifecycle,
+      status: "raw-expired",
+    },
+  };
+}
+
+function createRetentionContext(): RetentionContext {
+  return {
+    reviewSnapshots: new Map(),
+    triggerMaps: new Map(),
+  };
+}
+
+async function loadLegacyRetentionDays(
+  homeDir: string,
+  projectKey?: string,
+): Promise<Map<string, number>> {
+  const states = await listSessionMemoryStates({ homeDir, projectKey });
+  return new Map(
+    states.map((state) => [`${state.projectKey}\0${state.sessionKey}`, state.policy.retentionDays]),
+  );
+}
+
+function legacyExpiresAt(
+  segment: SessionEvidenceSegmentV1,
+  retentionDays: Map<string, number>,
+): string {
+  return addDays(
+    segment.createdAt,
+    retentionDays.get(sessionKey(segment)) ?? DEFAULT_LEGACY_RETENTION_DAYS,
+  );
+}
+
+function sessionKey(segment: SessionEvidenceSegmentV1): string {
+  return `${segment.projectKey}\0${segment.sessionKey}`;
+}
+
+function addDays(value: string, days: number): string {
+  const timestamp = Date.parse(value);
+  if (!Number.isFinite(timestamp)) throw new Error("Invalid Session Evidence retention timestamp.");
+  return new Date(timestamp + days * 24 * 60 * 60 * 1000).toISOString();
+}
+
+async function expireCapturedRawEvents(input: {
+  homeDir: string;
+  projectKey?: string;
+  now?: string | Date;
+  apply: boolean;
+}): Promise<{ payloads: number; rawBytes: number }> {
+  const now = Date.parse(normalizeTimestamp(input.now));
+  const states = await listSessionMemoryStates({
+    homeDir: input.homeDir,
+    projectKey: input.projectKey,
+  });
+  let payloads = 0;
+  let rawBytes = 0;
+  for (const state of states) {
+    const stateUpdatedAt = Date.parse(state.updatedAt);
+    if (Number.isFinite(stateUpdatedAt) && now - stateUpdatedAt < 5 * 60 * 1000) continue;
+    const paths = resolveSessionMemoryPaths({
+      homeDir: input.homeDir,
+      projectKey: state.projectKey,
+      sessionKey: state.sessionKey,
+    });
+    const cursor = await readCursor(paths.cursorPath);
+    const capturedLine = cursor?.lastCapturedLine ?? 0;
+    if (capturedLine <= 0) continue;
+    let text: string;
+    try {
+      text = await readFile(paths.eventsPath, "utf8");
+    } catch {
+      continue;
+    }
+    const lines = text.split("\n").filter((line) => line.trim() !== "");
+    let changed = false;
+    let filePayloads = 0;
+    let fileRawBytes = 0;
+    const nextLines = lines.map((line, index) => {
+      if (index >= capturedLine) return line;
+      let event: SessionMemoryRawEventV1;
+      try {
+        event = JSON.parse(line) as SessionMemoryRawEventV1;
+      } catch {
+        return line;
+      }
+      const expiresAt =
+        Date.parse(event.receivedAt) + state.policy.retentionDays * 24 * 60 * 60 * 1000;
+      if (
+        !Number.isFinite(expiresAt) ||
+        expiresAt > now ||
+        event.rawPayloadExpired === true ||
+        event.rawPayloadJson === "{}"
+      ) {
+        return line;
+      }
+      filePayloads += 1;
+      fileRawBytes += Buffer.byteLength(event.rawPayloadJson, "utf8");
+      if (!input.apply) return line;
+      changed = true;
+      return JSON.stringify({
+        ...event,
+        rawPayloadJson: "{}",
+        rawPayloadTruncated: false,
+        rawPayloadExpired: true,
+      } satisfies SessionMemoryRawEventV1);
+    });
+    if (!input.apply) {
+      payloads += filePayloads;
+      rawBytes += fileRawBytes;
+      continue;
+    }
+    if (changed && (await rewriteSessionRawEvents(paths.eventsPath, nextLines, text))) {
+      payloads += filePayloads;
+      rawBytes += fileRawBytes;
+    }
+  }
+  return { payloads, rawBytes };
+}
+
+async function readCursor(path: string): Promise<{ lastCapturedLine: number | null } | null> {
+  try {
+    return JSON.parse(await readFile(path, "utf8")) as { lastCapturedLine: number | null };
+  } catch {
+    return null;
+  }
+}