@evo-dev/core 0.0.1-alpha.11 → 0.0.1-alpha.12

package/src/evolution/evidence/session-memory/storage.ts

@@ -1,7 +1,20 @@
-import { appendFile, mkdir, readFile, readdir, writeFile } from "node:fs/promises";
+import { randomUUID } from "node:crypto";
+import {
+  appendFile,
+  chmod,
+  link,
+  lstat,
+  mkdir,
+  open,
+  readFile,
+  readdir,
+  rename,
+  rm,
+  writeFile,
+} from "node:fs/promises";
 import { dirname, join } from "node:path";
 import { resolveEvoDevPaths } from "../../../config/paths.ts";
-import { isNotFoundError } from "../../../utils/index.ts";
+import { isFileExistsError, isNotFoundError, sha256Hex } from "../../../utils/index.ts";
 import { resolveSessionMemoryPaths } from "./paths.ts";
 import { detectSessionMemorySensitivity, mergeSessionMemorySensitivity } from "./sensitivity.ts";
 import type {
@@ -268,11 +281,332 @@ export async function writeSessionIndex(
   });
 }
 
+export async function writeHistoricalSessionEvidenceSegment(input: {
+  homeDir: string;
+  paths: SessionMemoryPaths;
+  segment: SessionEvidenceSegmentV1;
+}): Promise<{ created: boolean; segment: SessionEvidenceSegmentV1 }> {
+  if (input.segment.origin?.kind !== "historical-import" || input.segment.retention === undefined) {
+    throw new Error("Historical Session Evidence requires origin and retention metadata.");
+  }
+  assertSessionMemoryPathsMatchSegment(input.homeDir, input.paths, input.segment);
+  await ensurePrivateSessionMemoryDirectory(input.paths.segmentsDir);
+  const path = input.paths.segmentPath(input.segment.id);
+  const created = await writePrivateImmutableText(
+    path,
+    `${JSON.stringify(input.segment, null, 2)}\n`,
+  );
+  if (created) {
+    return { created: true, segment: input.segment };
+  }
+  const existingInfo = await lstat(path);
+  if (!existingInfo.isFile() || existingInfo.isSymbolicLink()) {
+    throw new Error("Historical Session Evidence path is not a regular state file.");
+  }
+  const existing = await readSessionEvidenceSegment({
+    homeDir: input.homeDir,
+    projectKey: input.segment.projectKey,
+    sessionKey: input.segment.sessionKey,
+    segmentId: input.segment.id,
+  });
+  assertHistoricalSegmentCompatible(existing, input.segment);
+  await chmod(path, 0o600);
+  return { created: false, segment: existing };
+}
+
+export async function writeHistoricalSessionStateAndIndex(input: {
+  homeDir: string;
+  paths: SessionMemoryPaths;
+  projectKey: string;
+  sessionKey: string;
+  target: SessionMemoryStateV1["target"];
+  roleId: string | null;
+  policy: SessionMemoryStateV1["policy"];
+  now: string;
+}): Promise<{ state: SessionMemoryStateV1; segmentIds: string[] }> {
+  const expectedPaths = resolveSessionMemoryPaths({
+    homeDir: input.homeDir,
+    projectKey: input.projectKey,
+    sessionKey: input.sessionKey,
+  });
+  if (
+    expectedPaths.rootDir !== input.paths.rootDir ||
+    expectedPaths.projectDir !== input.paths.projectDir ||
+    expectedPaths.sessionDir !== input.paths.sessionDir ||
+    expectedPaths.segmentsDir !== input.paths.segmentsDir ||
+    expectedPaths.statePath !== input.paths.statePath ||
+    expectedPaths.indexPath !== input.paths.indexPath
+  ) {
+    throw new Error("Historical Session Memory state path does not match its identity.");
+  }
+  await ensurePrivateSessionMemoryDirectory(input.paths.rootDir);
+  await ensurePrivateSessionMemoryDirectory(input.paths.projectDir);
+  await ensurePrivateSessionMemoryDirectory(input.paths.sessionDir);
+  await ensurePrivateSessionMemoryDirectory(input.paths.segmentsDir);
+  const existing = await readSessionState(input.paths.statePath);
+  const segments = await readSessionSegmentsFromDirectory(input.paths.segmentsDir);
+  const ordered = segments.sort(
+    (left, right) =>
+      left.createdAt.localeCompare(right.createdAt) || left.id.localeCompare(right.id),
+  );
+  const last = ordered.at(-1) ?? null;
+  const state: SessionMemoryStateV1 = {
+    schemaVersion: 1,
+    kind: "session-memory-state",
+    projectKey: input.projectKey,
+    sessionKey: input.sessionKey,
+    target: input.target,
+    runId: null,
+    roleId: input.roleId,
+    initialized: true,
+    createdAt: existing?.createdAt ?? input.now,
+    updatedAt: input.now,
+    counters: existing?.counters ?? {
+      messageTokenEstimate: 0,
+      tokensSinceLastUpdate: 0,
+      toolCallsSinceLastUpdate: 0,
+      userPromptCount: 0,
+      assistantTurnCount: 0,
+      toolCallCount: 0,
+      failureSignalCount: 0,
+      verificationSignalCount: 0,
+    },
+    activeSpan: null,
+    pendingSignals: [],
+    lastSegmentId: last?.id ?? null,
+    lastMemoryUpdateAt: input.now,
+    policy: input.policy,
+  };
+  const index = {
+    schemaVersion: 1,
+    kind: "session-memory-index",
+    projectKey: input.projectKey,
+    sessionKey: input.sessionKey,
+    updatedAt: input.now,
+    lastSegmentId: last?.id ?? null,
+    segments: ordered.map((segment) => ({
+      id: segment.id,
+      reason: segment.reason,
+      strength: segment.strength,
+      createdAt: segment.createdAt,
+      reviewState: segment.lifecycle.reviewState,
+    })),
+  };
+  await writePrivateAtomicJson(input.paths.statePath, state);
+  await writePrivateAtomicJson(input.paths.indexPath, index);
+  return { state, segmentIds: ordered.map((segment) => segment.id) };
+}
+
+export async function writeSessionEvidenceSegmentAtomic(input: {
+  homeDir: string;
+  segment: SessionEvidenceSegmentV1;
+}): Promise<void> {
+  assertHistoricalRetentionState(input.segment);
+  const paths = resolveSessionMemoryPaths({
+    homeDir: input.homeDir,
+    projectKey: input.segment.projectKey,
+    sessionKey: input.segment.sessionKey,
+  });
+  await writePrivateAtomicJson(paths.segmentPath(input.segment.id), input.segment);
+}
+
+function assertHistoricalRetentionState(segment: SessionEvidenceSegmentV1): void {
+  if (segment.origin?.kind !== "historical-import" || segment.retention === undefined) {
+    throw new Error("Atomic Session Evidence rewrite is limited to historical retention.");
+  }
+  if (segment.retention.rawState === "available") {
+    if (
+      !segment.rawExcerpt.stored ||
+      segment.rawExcerpt.byteLength !== Buffer.byteLength(segment.rawExcerpt.content, "utf8") ||
+      segment.rawExcerpt.sha256 !== sha256Hex(segment.rawExcerpt.content) ||
+      segment.retention.originalRawSha256 !== segment.rawExcerpt.sha256
+    ) {
+      throw new Error("Available historical Session Evidence raw state is invalid.");
+    }
+    return;
+  }
+  if (
+    segment.rawExcerpt.stored ||
+    segment.rawExcerpt.content !== "" ||
+    segment.rawExcerpt.byteLength !== 0 ||
+    segment.rawExcerpt.sha256 !== sha256Hex("") ||
+    segment.retention.rawPurgedAt === null ||
+    segment.lifecycle.status !== "raw-expired"
+  ) {
+    throw new Error("Purged historical Session Evidence raw state is invalid.");
+  }
+}
+
 export async function writeJson(path: string, value: unknown): Promise<void> {
   await mkdir(dirname(path), { recursive: true });
   await writeFile(path, `${JSON.stringify(value, null, 2)}\n`, "utf8");
 }
 
+async function readSessionSegmentsFromDirectory(
+  segmentsDir: string,
+): Promise<SessionEvidenceSegmentV1[]> {
+  const segments: SessionEvidenceSegmentV1[] = [];
+  for (const path of await listJsonFiles(segmentsDir)) {
+    segments.push(parseSessionEvidenceSegment(JSON.parse(await readFile(path, "utf8"))));
+  }
+  return segments;
+}
+
+async function ensurePrivateSessionMemoryDirectory(path: string): Promise<void> {
+  await mkdir(path, { recursive: true, mode: 0o700 });
+  await chmod(path, 0o700);
+}
+
+async function writePrivateAtomicJson(path: string, value: unknown): Promise<void> {
+  await ensurePrivateSessionMemoryDirectory(dirname(path));
+  const temporaryPath = join(dirname(path), `.${randomUUID()}.tmp`);
+  try {
+    const handle = await open(temporaryPath, "wx", 0o600);
+    try {
+      await handle.writeFile(`${JSON.stringify(value, null, 2)}\n`, "utf8");
+      await handle.sync();
+    } finally {
+      await handle.close();
+    }
+    await rename(temporaryPath, path);
+    await chmod(path, 0o600);
+  } finally {
+    await rm(temporaryPath, { force: true }).catch(() => undefined);
+  }
+}
+
+async function writePrivateImmutableText(path: string, content: string): Promise<boolean> {
+  await ensurePrivateSessionMemoryDirectory(dirname(path));
+  const temporaryPath = join(dirname(path), `.${randomUUID()}.tmp`);
+  try {
+    const handle = await open(temporaryPath, "wx", 0o600);
+    try {
+      await handle.writeFile(content, "utf8");
+      await handle.sync();
+    } finally {
+      await handle.close();
+    }
+    try {
+      await link(temporaryPath, path);
+      await chmod(path, 0o600);
+      return true;
+    } catch (error) {
+      if (!isFileExistsError(error)) throw error;
+      return false;
+    }
+  } finally {
+    await rm(temporaryPath, { force: true }).catch(() => undefined);
+  }
+}
+
+function assertHistoricalSegmentCompatible(
+  existing: SessionEvidenceSegmentV1,
+  expected: SessionEvidenceSegmentV1,
+): void {
+  const existingOrigin = existing.origin;
+  const expectedOrigin = expected.origin;
+  const existingRetention = existing.retention;
+  const expectedRetention = expected.retention;
+  if (
+    existingOrigin?.kind !== "historical-import" ||
+    expectedOrigin?.kind !== "historical-import" ||
+    existingRetention === undefined ||
+    expectedRetention === undefined
+  ) {
+    throw new Error("Historical Session Evidence identity conflict.");
+  }
+  const existingProjection = {
+    schemaVersion: existing.schemaVersion,
+    kind: existing.kind,
+    id: existing.id,
+    projectKey: existing.projectKey,
+    runId: existing.runId,
+    roleId: existing.roleId,
+    sessionKey: existing.sessionKey,
+    target: existing.target,
+    createdAt: existing.createdAt,
+    reason: existing.reason,
+    strength: existing.strength,
+    origin: existingOrigin,
+    source: existing.source,
+    signals: existing.signals,
+    normalized: existing.normalized,
+    privacy: {
+      localOnly: existing.privacy.localOnly,
+      secretsDetected: existing.privacy.secretsDetected,
+      sensitivity: existing.privacy.sensitivity,
+      sensitivityReasons: existing.privacy.sensitivityReasons,
+      redactionApplied: existing.privacy.redactionApplied,
+      externalUploadAllowed: existing.privacy.externalUploadAllowed,
+    },
+    rawTruncated: existing.rawExcerpt.truncated,
+    originalRawSha256: existingRetention.originalRawSha256,
+    retentionPolicyDays: existingRetention.policyDays,
+    retentionExpiresAt: existingRetention.expiresAt,
+  };
+  const expectedProjection = {
+    schemaVersion: expected.schemaVersion,
+    kind: expected.kind,
+    id: expected.id,
+    projectKey: expected.projectKey,
+    runId: expected.runId,
+    roleId: expected.roleId,
+    sessionKey: expected.sessionKey,
+    target: expected.target,
+    createdAt: expected.createdAt,
+    reason: expected.reason,
+    strength: expected.strength,
+    origin: expectedOrigin,
+    source: expected.source,
+    signals: expected.signals,
+    normalized: expected.normalized,
+    privacy: {
+      localOnly: expected.privacy.localOnly,
+      secretsDetected: expected.privacy.secretsDetected,
+      sensitivity: expected.privacy.sensitivity,
+      sensitivityReasons: expected.privacy.sensitivityReasons,
+      redactionApplied: expected.privacy.redactionApplied,
+      externalUploadAllowed: expected.privacy.externalUploadAllowed,
+    },
+    rawTruncated: expected.rawExcerpt.truncated,
+    originalRawSha256: expectedRetention.originalRawSha256,
+    retentionPolicyDays: expectedRetention.policyDays,
+    retentionExpiresAt: expectedRetention.expiresAt,
+  };
+  if (
+    JSON.stringify(existingProjection) !== JSON.stringify(expectedProjection) ||
+    (existingRetention.rawState === "available" &&
+      (!existing.rawExcerpt.stored ||
+        existing.rawExcerpt.encoding !== "utf8" ||
+        existing.rawExcerpt.byteLength !== Buffer.byteLength(existing.rawExcerpt.content, "utf8") ||
+        existing.rawExcerpt.sha256 !== sha256Hex(existing.rawExcerpt.content) ||
+        existing.rawExcerpt.sha256 !== expected.rawExcerpt.sha256))
+  ) {
+    throw new Error("Historical Session Evidence immutable content conflict.");
+  }
+}
+
+function assertSessionMemoryPathsMatchSegment(
+  homeDir: string,
+  paths: SessionMemoryPaths,
+  segment: SessionEvidenceSegmentV1,
+): void {
+  const expected = resolveSessionMemoryPaths({
+    homeDir,
+    projectKey: segment.projectKey,
+    sessionKey: segment.sessionKey,
+  });
+  if (
+    paths.rootDir !== expected.rootDir ||
+    paths.projectDir !== expected.projectDir ||
+    paths.sessionDir !== expected.sessionDir ||
+    paths.segmentsDir !== expected.segmentsDir ||
+    paths.segmentPath(segment.id) !== expected.segmentPath(segment.id)
+  ) {
+    throw new Error("Historical Session Evidence path does not match its identity.");
+  }
+}
+
 async function countJsonlLines(path: string): Promise<number> {
   try {
     const text = await readFile(path, "utf8");

package/src/evolution/evidence/session-memory/types.ts

@@ -7,7 +7,8 @@ export type SessionMemorySegmentReason =
   | "failure-signal"
   | "permission-denied"
   | "verification-after-fix"
-  | "explicit-memory-intent";
+  | "explicit-memory-intent"
+  | "historical-import";
 export type SessionMemorySignalStrength = "normal" | "strong";
 export type SessionMemorySensitivity = "safe-metadata" | "private-content" | "credential";
 export type SessionMemorySensitivityReason =
@@ -92,6 +93,68 @@ export interface SessionMemoryCursorV1 {
   updatedAt: string;
 }
 
+export interface HistoricalImportOriginV1 {
+  kind: "historical-import";
+  importId: string;
+  generationId: string;
+  snapshotId: string;
+  sourceKey: string;
+  firstRecordKeyHash: string;
+  lastRecordKeyHash: string;
+  recordCount: number;
+}
+
+export interface HistoricalImportRetentionV1 {
+  policyDays: number;
+  expiresAt: string;
+  rawState: "available" | "purged";
+  rawPurgedAt: string | null;
+  originalRawSha256: string;
+}
+
+export interface HistoricalImportStoredRecordV1 {
+  schemaVersion: 1;
+  kind: "historical-import-record";
+  recordKeyHash: string;
+  recordType: "user" | "reasoning" | "assistant" | "assistant-tool-call" | "tool";
+  canonicalJson: string;
+  canonicalJsonTruncated: boolean;
+}
+
+export interface SemanticEvidencePacketV1 {
+  schemaVersion: 1;
+  kind: "semantic-evidence-packet";
+  projectKey: string;
+  segmentId: string;
+  conversation: Array<{
+    role: "user" | "assistant";
+    text: string;
+    recordKeyHash: string;
+  }>;
+  toolEvidence: Array<{
+    reason: "failure" | "verification" | "permission" | "rollback";
+    toolName: string;
+    safeInputExcerpt: string | null;
+    outputExcerpt: string | null;
+    status: string | null;
+    exitCode: number | null;
+    inputTruncated: boolean;
+    outputTruncated: boolean;
+  }>;
+  touchedPaths: string[];
+  selection: {
+    selectedRecords: number;
+    droppedRecords: number;
+    redactionCount: number;
+    truncated: boolean;
+  };
+  privacy: {
+    containsPrivateContent: true;
+    credentialRedacted: true;
+    rawSegmentIncluded: false;
+  };
+}
+
 export interface SessionEvidenceSegmentV1 {
   schemaVersion: 1;
   kind: "session-evidence-segment";
@@ -104,6 +167,8 @@ export interface SessionEvidenceSegmentV1 {
   createdAt: string;
   reason: SessionMemorySegmentReason;
   strength: SessionMemorySignalStrength;
+  origin?: HistoricalImportOriginV1;
+  retention?: HistoricalImportRetentionV1;
   source: {
     traceRefId: string | null;
     sourcePath: string | null;
@@ -116,7 +181,7 @@ export interface SessionEvidenceSegmentV1 {
   };
   signals: SessionMemorySignal[];
   rawExcerpt: {
-    stored: true;
+    stored: boolean;
     encoding: "utf8";
     content: string;
     truncated: boolean;
@@ -144,7 +209,14 @@ export interface SessionEvidenceSegmentV1 {
     externalUploadAllowed: false;
   };
   lifecycle: {
-    status: "captured" | "pending-review" | "reviewed" | "distilled" | "ignored" | "deleted";
+    status:
+      | "captured"
+      | "pending-review"
+      | "reviewed"
+      | "distilled"
+      | "ignored"
+      | "deleted"
+      | "raw-expired";
     reviewState: "not-required" | "unreviewed" | "accepted" | "rejected" | "deferred";
     consumedByBatchIds: string[];
   };